diff --git "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" --- "a/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" +++ "b/wandb/run-20231119_105908-nwx0xdmm/logs/debug-internal.log" @@ -1,81498 +1,3 @@ -2023-11-19 10:59:09,032 INFO StreamThr :607337 [internal.py:wandb_internal():86] W&B internal server running at pid: 607337, started at: 2023-11-19 10:59:09.030377 -2023-11-19 10:59:09,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status -2023-11-19 10:59:09,054 INFO WriterThread:607337 [datastore.py:open_for_write():85] open: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb -2023-11-19 10:59:09,056 DEBUG SenderThread:607337 [sender.py:send():380] send: header -2023-11-19 10:59:09,057 DEBUG SenderThread:607337 [sender.py:send():380] send: run -2023-11-19 10:59:09,497 INFO SenderThread:607337 [dir_watcher.py:__init__():211] watching files in: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files -2023-11-19 10:59:09,507 INFO SenderThread:607337 [sender.py:_start_run_threads():1124] run started: nwx0xdmm with start time 1700384349.034087 -2023-11-19 10:59:09,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: check_version -2023-11-19 10:59:09,508 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: check_version -2023-11-19 10:59:09,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: run_start -2023-11-19 10:59:09,570 DEBUG HandlerThread:607337 [system_info.py:__init__():32] System info init -2023-11-19 10:59:09,570 DEBUG HandlerThread:607337 [system_info.py:__init__():47] System info init done -2023-11-19 10:59:09,570 INFO HandlerThread:607337 [system_monitor.py:start():194] Starting system monitor -2023-11-19 10:59:09,571 INFO SystemMonitor:607337 [system_monitor.py:_start():158] Starting system asset monitoring threads -2023-11-19 10:59:09,571 INFO HandlerThread:607337 [system_monitor.py:probe():214] Collecting system info -2023-11-19 10:59:09,572 INFO SystemMonitor:607337 [interfaces.py:start():190] Started cpu monitoring -2023-11-19 10:59:09,573 INFO SystemMonitor:607337 [interfaces.py:start():190] Started disk monitoring -2023-11-19 10:59:09,574 INFO SystemMonitor:607337 [interfaces.py:start():190] Started gpu monitoring -2023-11-19 10:59:09,574 INFO SystemMonitor:607337 [interfaces.py:start():190] Started memory monitoring -2023-11-19 10:59:09,576 INFO SystemMonitor:607337 [interfaces.py:start():190] Started network monitoring -2023-11-19 10:59:09,581 DEBUG HandlerThread:607337 [system_info.py:probe():196] Probing system -2023-11-19 10:59:09,678 DEBUG HandlerThread:607337 [gitlib.py:_init_repo():56] git repository is invalid -2023-11-19 10:59:09,678 DEBUG HandlerThread:607337 [system_info.py:probe():244] Probing system done -2023-11-19 10:59:09,678 DEBUG HandlerThread:607337 [system_monitor.py:probe():223] {'os': 'Linux-5.15.123.1-microsoft-standard-WSL2-x86_64-with-glibc2.35', 'python': '3.10.13', 'heartbeatAt': '2023-11-19T08:59:09.582026', 'startedAt': '2023-11-19T08:59:08.979379', 'docker': None, 'cuda': None, 'args': ('--local_rank=0', '--deepspeed=ds_config.json', '--model_name_or_path=openai/whisper-large-v3', '--train_dataset_names', 'mozilla-foundation/common_voice_11_0', 'mozilla-foundation/common_voice_11_0', 'google/fleurs', 'Finnish-NLP/css10_audio_processed', 'Finnish-NLP/youtube_fi_110k_preprocessed', 'Finnish-NLP/voxpopuli_fi_audio_processed', 'Finnish-NLP/parliament_audio_processed', 'Finnish-NLP/aalto_eduskunta_asr_audio_processed', 'Finnish-NLP/fbc_dialog_audio_processed', 'Finnish-NLP/fbc_monolog_audio_processed', '--train_dataset_config_names', 'fi', 'fi', 'fi_fi', '', '', '', '', '', '', '', '--train_dataset_split_name', 'train', 'validation', 'train', 'train', 'train', 'train', 'train', 'train', 'train', 'train', '--train_dataset_text_column_names', 'sentence', 'sentence', 'raw_transcription', 'sentence', 'texts', 'sentence', 'sentence', 'sentence', 'sentence', 'sentence', '--eval_dataset_name=mozilla-foundation/common_voice_11_0', '--eval_dataset_config_name=fi', '--eval_split_name=test', '--language=finnish', '--model_index_name=WhisperLargeV3Finnish', '--max_steps=60000', '--output_dir=./', '--per_device_train_batch_size=8', '--per_device_eval_batch_size=4', '--gradient_accumulation_steps=2', '--logging_steps=20', '--learning_rate=3e-6', '--warmup_steps=1000', '--evaluation_strategy=steps', '--eval_steps=1000', '--save_strategy=steps', '--save_steps=1000', '--generation_max_length=225', '--length_column_name=input_length', '--max_duration_in_seconds=30', '--text_column_name=sentence', '--freeze_feature_encoder=False', '--report_to=wandb', '--metric_for_best_model=wer', '--greater_is_better=False', '--load_best_model_at_end', '--gradient_checkpointing', '--fp16', '--torch_compile', '--overwrite_output_dir', '--do_train', '--do_eval', '--predict_with_generate', '--do_normalize_eval', '--use_auth_token', '--push_to_hub', '--push_to_hub_model_id=WhisperLargeFinnishV3', '--resume_from_checkpoint=./checkpoint-7000'), 'state': 'running', 'program': '/mnt/e/run_speech_recognition_seq2seq_streaming.py', 'codePathLocal': 'run_speech_recognition_seq2seq_streaming.py', 'codePath': 'run_speech_recognition_seq2seq_streaming.py', 'host': 'DESKTOP-59O9VN1', 'username': 'rasmus', 'executable': '/home/rasmus/miniconda3/envs/WhisperFinetuneEnv/bin/python', 'cpu_count': 4, 'cpu_count_logical': 8, 'cpu_freq': {'current': 3911.9949999999994, 'min': 0.0, 'max': 0.0}, 'cpu_freq_per_core': [{'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}, {'current': 3911.995, 'min': 0.0, 'max': 0.0}], 'disk': {'/': {'total': 1006.853931427002, 'used': 37.43992614746094}}, 'gpu': 'NVIDIA GeForce RTX 4080', 'gpu_count': 1, 'gpu_devices': [{'name': 'NVIDIA GeForce RTX 4080', 'memory_total': 17171480576}], 'memory': {'total': 54.92573928833008}} -2023-11-19 10:59:09,679 INFO HandlerThread:607337 [system_monitor.py:probe():224] Finished collecting system info -2023-11-19 10:59:09,679 INFO HandlerThread:607337 [system_monitor.py:probe():227] Publishing system info -2023-11-19 10:59:09,679 DEBUG HandlerThread:607337 [system_info.py:_save_pip():52] Saving list of pip packages installed into the current environment -2023-11-19 10:59:09,682 DEBUG HandlerThread:607337 [system_info.py:_save_pip():68] Saving pip packages done -2023-11-19 10:59:09,683 DEBUG HandlerThread:607337 [system_info.py:_save_conda():75] Saving list of conda packages installed into the current environment -2023-11-19 10:59:10,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_created():271] file/dir created: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/conda-environment.yaml -2023-11-19 10:59:10,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_created():271] file/dir created: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/requirements.txt -2023-11-19 10:59:14,432 DEBUG HandlerThread:607337 [system_info.py:_save_conda():87] Saving conda packages done -2023-11-19 10:59:14,448 INFO HandlerThread:607337 [system_monitor.py:probe():229] Finished publishing system info -2023-11-19 10:59:14,462 DEBUG SenderThread:607337 [sender.py:send():380] send: files -2023-11-19 10:59:14,463 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-metadata.json with policy now -2023-11-19 10:59:14,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:14,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 10:59:14,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 10:59:14,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 10:59:14,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/conda-environment.yaml -2023-11-19 10:59:14,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_created():271] file/dir created: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-metadata.json -2023-11-19 10:59:14,772 DEBUG SenderThread:607337 [sender.py:send():380] send: telemetry -2023-11-19 10:59:14,772 DEBUG SenderThread:607337 [sender.py:send():380] send: config -2023-11-19 10:59:14,773 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 10:59:14,774 DEBUG SenderThread:607337 [sender.py:send():380] send: telemetry -2023-11-19 10:59:14,774 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 10:59:14,775 WARNING SenderThread:607337 [sender.py:send_metric():1331] Seen metric with glob (shouldn't happen) -2023-11-19 10:59:14,775 DEBUG SenderThread:607337 [sender.py:send():380] send: telemetry -2023-11-19 10:59:15,137 INFO wandb-upload_0:607337 [upload_job.py:push():131] Uploaded file /tmp/tmpb0h5x3xmwandb/uc3dmej7-wandb-metadata.json -2023-11-19 10:59:15,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_created():271] file/dir created: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 10:59:17,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 10:59:20,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:21,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 10:59:25,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:29,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 10:59:29,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 10:59:29,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 10:59:30,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:35,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:40,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:41,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/config.yaml -2023-11-19 10:59:44,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 10:59:44,481 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 10:59:44,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 10:59:46,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:51,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 10:59:59,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 10:59:59,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 10:59:59,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:00:01,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:06,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:09,578 DEBUG SystemMonitor:607337 [system_monitor.py:_start():172] Starting system metrics aggregation loop -2023-11-19 11:00:09,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:00:12,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:14,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:00:14,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:00:14,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:00:17,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:22,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:27,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:29,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:00:29,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:00:29,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:00:33,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:38,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:39,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:00:44,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:00:44,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:00:44,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:00:44,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:49,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:54,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:00:59,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:00:59,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:00:59,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:01:00,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:05,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:09,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:01:11,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:14,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:01:14,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:01:14,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:01:16,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:21,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:01:22,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:27,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:29,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:01:29,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:01:29,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:01:32,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:37,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:39,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:01:43,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:44,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:01:44,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:01:44,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:01:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:53,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:58,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:01:59,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:01:59,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:01:59,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:02:04,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:09,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:02:10,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:14,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:02:14,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:02:14,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:02:15,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:20,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:25,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:29,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:02:29,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:02:29,497 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:02:30,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:35,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:39,582 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:02:41,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:44,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:02:44,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:02:44,503 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:02:46,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:51,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:56,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:02:59,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:02:59,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:02:59,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:03:01,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:06,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:09,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:03:12,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:14,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:03:14,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:03:14,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:03:17,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:22,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:27,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:29,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:03:29,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:03:29,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:03:32,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:37,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:39,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:03:43,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:44,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:03:44,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:03:44,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:03:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:53,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:58,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:03:59,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:03:59,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:03:59,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:04:04,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:09,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:04:10,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:14,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:04:14,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:04:14,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:04:15,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:20,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:25,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:29,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:04:29,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:04:29,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:04:30,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:35,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:39,585 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:04:41,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:44,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:04:44,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:04:44,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:04:46,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:51,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:56,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:04:59,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:04:59,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:04:59,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:05:01,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:06,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:09,586 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:05:12,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:14,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:05:14,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:05:14,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:05:17,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:22,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:27,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:29,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:05:29,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:05:29,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:05:32,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:37,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:39,594 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:05:43,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:44,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:05:44,508 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:05:44,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:05:48,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:53,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:58,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:05:59,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:05:59,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:05:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:06:04,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:09,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:06:10,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:14,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:06:14,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:06:14,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:06:15,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:20,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:25,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:29,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:06:29,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:06:29,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:06:31,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:36,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:39,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:06:42,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:44,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:06:44,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:06:44,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:06:47,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:52,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:57,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:06:59,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:06:59,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:06:59,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:07:02,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:07,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:09,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:07:13,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:14,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:07:14,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:07:14,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:07:18,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:23,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:28,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:29,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:07:29,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:07:29,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:07:33,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:38,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:39,597 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:07:44,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:44,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:07:44,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:07:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:07:49,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:54,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:07:59,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:07:59,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:07:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:07:59,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:04,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:09,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:08:10,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:14,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:08:14,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:08:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:08:15,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:20,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:25,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:29,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:08:29,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:08:29,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:08:30,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:35,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:39,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:08:41,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:44,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:08:44,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:08:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:08:46,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:51,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:56,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:08:59,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:08:59,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:08:59,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:09:02,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:07,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:09,599 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:09:13,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:14,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:09:14,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:09:14,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:09:18,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:23,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:28,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:29,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:09:29,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:09:29,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:09:33,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:38,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:39,599 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:09:44,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:44,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:09:44,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:09:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:09:49,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:54,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:09:59,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:09:59,512 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:09:59,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:09:59,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:04,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:09,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:10:10,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:14,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:10:14,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:10:14,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:10:15,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:20,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:25,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:29,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:10:29,516 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:10:29,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:10:30,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:35,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:39,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:10:41,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:44,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:10:44,516 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:10:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:10:46,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:51,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:56,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:10:59,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:10:59,516 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:10:59,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:11:02,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:07,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:09,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:11:13,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:14,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:11:14,534 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:11:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:11:18,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:23,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:28,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:29,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:11:29,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:11:29,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:11:34,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:39,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:11:40,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:44,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:11:44,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:11:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:11:45,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:50,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:55,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:11:59,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:11:59,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:11:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:12:01,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:06,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:09,604 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:12:12,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:14,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:12:14,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:12:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:12:17,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:22,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:27,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:29,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:12:29,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:12:29,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:12:33,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:39,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:12:44,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:44,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:12:44,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:12:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:12:49,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:54,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:12:59,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:12:59,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:12:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:12:59,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:04,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:09,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:13:10,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:14,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:13:14,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:13:14,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:13:15,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:20,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:25,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:29,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:13:29,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:13:29,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:13:30,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:35,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:39,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:13:41,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:44,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:13:44,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:13:44,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:13:46,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:51,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:56,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:13:59,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:13:59,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:13:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:14:02,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:07,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:09,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:14:13,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:14,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:14:14,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:14:14,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:14:18,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:23,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:28,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:29,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:14:29,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:14:29,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:14:33,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:38,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:39,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:14:44,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:44,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:14:44,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:14:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:14:44,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:14:49,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:54,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:14:59,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:14:59,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:14:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:15:00,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:05,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:09,608 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:15:11,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:14,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:15:14,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:15:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:15:16,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:21,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:26,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:29,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:15:29,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:15:29,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:15:31,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:36,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:39,608 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:15:42,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:44,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:15:44,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:15:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:15:47,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:52,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:57,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:15:59,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:15:59,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:15:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:16:03,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:08,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:09,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:16:14,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:14,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:16:14,521 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:16:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:16:19,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:24,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:29,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:16:29,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:16:29,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:16:30,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:35,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:39,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:16:41,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:44,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:16:44,521 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:16:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:16:46,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:51,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:56,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:16:59,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:16:59,522 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:16:59,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:17:02,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:07,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:09,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:17:13,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:14,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:17:14,521 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:17:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:17:18,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:23,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:28,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:29,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:17:29,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:17:29,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:17:34,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:39,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:17:40,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:44,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:17:44,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:17:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:17:45,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:50,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:55,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:17:59,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:17:59,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:17:59,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:18:00,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:05,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:09,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:18:11,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:14,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:18:14,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:18:14,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:18:16,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:21,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:26,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:29,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:18:29,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:18:29,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:18:32,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:37,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:39,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:18:43,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:44,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:18:44,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:18:44,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:18:48,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:53,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:58,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:18:58,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:18:59,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:18:59,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:18:59,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:19:04,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:09,614 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:19:10,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:14,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:19:14,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:19:14,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:19:15,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:20,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:25,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:29,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:19:29,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:19:29,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:19:30,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:35,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:39,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:19:41,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:44,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:19:44,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:19:44,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:19:46,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:51,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:56,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:19:59,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:19:59,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:19:59,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:20:02,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:07,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:09,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:20:13,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:14,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:20:14,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:20:14,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:20:18,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:23,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:28,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:29,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:20:29,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:20:29,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:20:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:38,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:39,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:20:44,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:44,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:20:44,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:20:44,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:20:49,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:54,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:20:59,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:20:59,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:20:59,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:20:59,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:04,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:09,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:21:10,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:14,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:21:14,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:21:14,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:21:15,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:20,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:25,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:29,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:21:29,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:21:29,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:21:30,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:35,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:39,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:21:41,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:44,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:21:44,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:21:44,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:21:46,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:51,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:56,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:21:59,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:21:59,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:21:59,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:22:01,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:06,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:09,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:22:12,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:14,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:22:14,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:22:14,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:22:17,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:22,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:27,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:29,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:22:29,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:22:29,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:22:33,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:38,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:39,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:22:44,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:44,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:22:44,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:22:44,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:22:49,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:54,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:22:59,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:22:59,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:22:59,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:22:59,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:04,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:09,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:23:10,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:14,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:23:14,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:23:14,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:23:15,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:20,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:25,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:29,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:23:29,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:23:29,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:23:31,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:36,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:39,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:23:42,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:44,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:23:44,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:23:44,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:23:47,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:52,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:57,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:23:59,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:23:59,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:23:59,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:24:02,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:07,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:09,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:24:13,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:24:14,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:24:14,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:24:18,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:23,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:28,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:29,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:24:29,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:24:29,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:24:34,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:39,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:24:40,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:44,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:24:44,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:24:44,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:24:45,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:50,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:55,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:24:59,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:24:59,557 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:24:59,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:25:00,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:05,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:09,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:25:11,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:14,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:25:14,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:25:14,558 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:25:16,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:21,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:26,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:29,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:25:29,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:25:29,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:25:31,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:36,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:39,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:25:42,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:44,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:25:44,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:25:44,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:25:47,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:52,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:57,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:25:59,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:25:59,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:25:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:26:02,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:07,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:09,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:26:13,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:14,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:26:14,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:26:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:26:18,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:23,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:28,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:29,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:26:29,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:26:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:26:34,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:39,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:26:40,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:44,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:26:44,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:26:44,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:26:45,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:50,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:55,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:26:59,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:26:59,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:26:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:27:01,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:06,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:09,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:27:12,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:14,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:27:14,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:27:14,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:27:17,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:22,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:27,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:29,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:27:29,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:27:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:27:33,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:38,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:39,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:27:44,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:44,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:27:44,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:27:44,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:27:49,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:54,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:27:59,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:27:59,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:27:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:27:59,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:04,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:09,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:28:10,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:14,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:28:14,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:28:14,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:28:15,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:20,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:25,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:29,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:28:29,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:28:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:28:30,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:35,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:39,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:28:41,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:44,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:28:44,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:28:44,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:28:46,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:51,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:56,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:28:59,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:28:59,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:28:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:29:02,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:07,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:09,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:29:12,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:14,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:29:14,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:29:14,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:29:17,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:22,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:27,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:29,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:29:29,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:29:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:29:33,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:38,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:39,634 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:29:44,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:44,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:29:44,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:29:44,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:29:45,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:29:49,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:54,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:29:59,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:29:59,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:29:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:30:00,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:05,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:09,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:30:11,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:14,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:30:14,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:30:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:30:16,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:21,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:26,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:29,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:30:29,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:30:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:30:31,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:36,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:39,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:30:42,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:44,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:30:44,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:30:44,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:30:47,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:52,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:57,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:30:59,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:30:59,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:30:59,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:31:02,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:07,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:09,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:31:13,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:14,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:31:14,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:31:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:31:18,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:23,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:28,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:29,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:31:29,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:31:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:31:33,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:38,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:39,637 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:31:44,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:44,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:31:44,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:31:44,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:31:49,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:54,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:31:59,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:31:59,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:31:59,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:32:00,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:05,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:09,637 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:32:11,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:14,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:32:14,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:32:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:32:16,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:21,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:26,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:29,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:32:29,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:32:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:32:32,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:37,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:39,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:32:43,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:44,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:32:44,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:32:44,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:32:48,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:53,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:58,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:32:59,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:32:59,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:32:59,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:33:04,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:09,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:33:10,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:14,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:33:14,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:33:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:33:15,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:20,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:25,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:29,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:33:29,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:33:29,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:33:31,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:36,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:39,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:33:42,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:44,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:33:44,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:33:44,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:33:47,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:52,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:57,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:33:59,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:33:59,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:33:59,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:34:03,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:08,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:09,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:34:14,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:14,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:34:14,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:34:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:34:19,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:24,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:29,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:34:29,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:34:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:34:29,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:34,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:39,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:34:40,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:44,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:34:44,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:34:44,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:34:45,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:50,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:55,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:34:59,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:34:59,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:34:59,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:35:00,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:05,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:09,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:35:11,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:14,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:35:14,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:35:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:35:16,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:21,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:26,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:29,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:35:29,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:35:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:35:32,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:37,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:39,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:35:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:44,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:35:44,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:35:44,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:35:48,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:53,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:58,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:35:59,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:35:59,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:35:59,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:36:04,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:09,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:36:10,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:14,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:36:14,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:36:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:36:15,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:20,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:25,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:29,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:36:29,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:36:29,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:36:31,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:36,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:39,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:36:42,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:44,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:36:44,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:36:44,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:36:47,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:52,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:57,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:36:59,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:36:59,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:36:59,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:37:02,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:07,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:09,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:37:13,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:14,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:37:14,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:37:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:37:18,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:23,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:28,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:29,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:37:29,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:37:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:37:34,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:39,644 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:37:40,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:44,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:37:44,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:37:44,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:37:45,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:50,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:55,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:37:59,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:37:59,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:37:59,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:38:01,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:06,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:09,645 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:38:12,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:14,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:38:14,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:38:14,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:38:17,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:22,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:28,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:29,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:38:29,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:38:29,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:38:29,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:38:33,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:38,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:39,645 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:38:44,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:44,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:38:44,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:38:44,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:38:49,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:54,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:38:59,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:38:59,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:38:59,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:38:59,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:04,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:09,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:39:10,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:14,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:39:14,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:39:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:39:15,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:25,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:29,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:39:29,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:39:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:39:31,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:36,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:39,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:39:42,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:44,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:39:44,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:39:44,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:39:47,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:52,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:39:59,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:39:59,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:39:59,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:40:02,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:07,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:09,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:40:13,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:14,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:40:14,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:40:14,601 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:40:18,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:23,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:28,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:29,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:40:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:40:29,601 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:40:33,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:38,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:39,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:40:44,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:44,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:40:44,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:40:44,602 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:40:49,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:54,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:40:59,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:40:59,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:40:59,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:40:59,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:04,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:09,648 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:41:10,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:14,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:41:14,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:41:14,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:41:15,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:20,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:25,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:29,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:41:29,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:41:29,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:41:30,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:35,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:39,660 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:41:41,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:44,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:41:44,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:41:44,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:41:46,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:51,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:56,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:41:59,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:41:59,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:41:59,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:42:01,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:06,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:09,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:42:12,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:14,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:42:14,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:42:14,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:42:17,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:22,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:27,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:42:29,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:42:29,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:42:33,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:38,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:39,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:42:44,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:44,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:42:44,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:42:44,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:42:49,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:54,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:42:59,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:42:59,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:42:59,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:43:00,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:05,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:09,651 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:43:11,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:14,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:43:14,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:43:14,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:43:16,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:21,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:26,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:43:29,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:43:29,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:43:32,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:37,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:39,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:43:43,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:44,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:43:44,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:43:44,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:43:48,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:53,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:58,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:43:59,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:43:59,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:43:59,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:44:03,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:08,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:09,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:44:14,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:14,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:44:14,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:44:14,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:44:18,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:44:20,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:25,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:44:29,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:44:29,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:44:30,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:35,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:39,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:44:41,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:44,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:44:44,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:44:44,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:44:46,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:51,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:56,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:44:59,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:44:59,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:44:59,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:45:02,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:07,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:09,660 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:45:13,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:14,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:45:14,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:45:14,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:45:18,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:23,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:28,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:29,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:45:29,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:45:29,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:45:34,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:39,654 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:45:40,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:44,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:45:44,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:45:44,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:45:45,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:50,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:55,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:45:59,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:45:59,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:45:59,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:46:01,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:06,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:09,654 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:46:12,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:14,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:46:14,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:46:14,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:46:17,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:22,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:27,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:29,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:46:29,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:46:29,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:46:33,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:38,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:39,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:46:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:44,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:46:44,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:46:44,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:46:49,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:54,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:46:59,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:46:59,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:46:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:47:00,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:05,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:09,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:47:11,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:14,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:47:14,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:47:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:47:16,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:21,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:26,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:29,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:47:29,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:47:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:47:32,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:37,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:39,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:47:43,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:44,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:47:44,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:47:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:47:48,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:53,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:58,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:47:59,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:47:59,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:47:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:48:03,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:08,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:09,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:48:14,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:14,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:48:14,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:48:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:48:19,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:24,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:29,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:48:29,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:48:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:48:29,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:34,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:39,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:48:40,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:44,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:48:44,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:48:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:48:45,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:50,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:55,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:48:59,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:48:59,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:48:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:49:01,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:06,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:09,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:49:12,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:14,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:49:14,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:49:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:49:17,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:22,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:27,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:29,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:49:29,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:49:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:49:33,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:38,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:39,660 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:49:44,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:44,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:49:44,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:49:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:49:49,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:54,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:49:59,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:49:59,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:49:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:50:00,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:05,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:09,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:50:11,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:14,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:50:14,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:50:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:50:16,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:21,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:26,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:29,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:50:29,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:50:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:50:32,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:37,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:39,660 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:50:43,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:44,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:50:44,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:50:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:50:48,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:53,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:58,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:50:59,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:50:59,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:50:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:51:04,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:09,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:09,661 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:51:14,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:14,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:51:14,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:51:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:51:19,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:24,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:29,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:51:29,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:51:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:51:30,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:35,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:39,661 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:51:41,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:44,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:51:44,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:51:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:51:46,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:51,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:56,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:51:59,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:51:59,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:51:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:52:02,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:07,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:09,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:52:13,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:14,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:52:14,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:52:14,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:52:18,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:23,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:28,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:29,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:52:29,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:52:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:52:34,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:39,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:52:40,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:44,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:52:44,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:52:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:52:45,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:50,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:55,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:52:59,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:52:59,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:52:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:53:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:05,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:09,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:53:11,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:14,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:53:14,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:53:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:53:16,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:21,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:26,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:29,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:53:29,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:53:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:53:32,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:37,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:39,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:53:43,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:44,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:53:44,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:53:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:53:48,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:53,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:58,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:53:59,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:53:59,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:53:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:54:03,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:08,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:09,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:54:14,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:14,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:54:14,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:54:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:54:19,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:24,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:29,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:54:29,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:54:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:54:30,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:35,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:39,664 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:54:41,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:44,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:54:44,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:54:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:54:46,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:51,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:56,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:54:59,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:54:59,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:54:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:55:02,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:07,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:09,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:55:13,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:14,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:55:14,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:55:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:55:18,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:23,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:28,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:29,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:55:29,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:55:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:55:33,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:38,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:39,675 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:55:44,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:44,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:55:44,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:55:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:55:49,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:54,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:55:59,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:55:59,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:55:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:56:00,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:05,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:09,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:56:11,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:14,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:56:14,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:56:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:56:16,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:21,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:26,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:29,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:56:29,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:56:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:56:31,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:36,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:39,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:56:42,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:44,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:56:44,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:56:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:56:47,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:52,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:57,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:56:59,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:56:59,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:56:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:57:02,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:07,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:09,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:57:13,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:14,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:57:14,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:57:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:57:18,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:23,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:28,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:29,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:57:29,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:57:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:57:34,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:39,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:57:40,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:44,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:57:44,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:57:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:57:45,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:50,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:55,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:57:59,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:57:59,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:57:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:58:01,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:58:01,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:06,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:09,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:58:12,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:14,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:58:14,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:58:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:58:17,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:22,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:27,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:29,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:58:29,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:58:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:58:33,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:38,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:39,679 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:58:41,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 11:58:44,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:44,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:58:44,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:58:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:58:49,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:54,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:58:59,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:58:59,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:58:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:58:59,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:04,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:09,679 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:59:10,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:14,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:59:14,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:59:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:59:15,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:20,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:25,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:29,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:59:29,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:59:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:59:30,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:35,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:39,679 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 11:59:41,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:44,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:59:44,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:59:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 11:59:46,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:51,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:56,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 11:59:59,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 11:59:59,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 11:59:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:00:02,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:07,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:09,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:00:13,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:14,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:00:14,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:00:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:00:18,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:23,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:28,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:29,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:00:29,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:00:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:00:34,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:39,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:00:40,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:44,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:00:44,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:00:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:00:45,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:50,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:55,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:00:59,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:00:59,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:00:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:01:00,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:05,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:09,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:01:11,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:14,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:01:14,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:01:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:01:16,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:26,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:29,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:01:29,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:01:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:01:32,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:37,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:39,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:01:43,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:44,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:01:44,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:01:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:01:48,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:53,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:58,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:01:59,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:01:59,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:01:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:02:04,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:09,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:02:10,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:14,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:02:14,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:02:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:02:15,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:20,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:25,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:29,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:02:29,621 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:02:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:02:30,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:35,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:39,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:02:41,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:44,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:02:44,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:02:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:02:46,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:51,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:56,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:02:59,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:02:59,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:02:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:03:02,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:07,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:09,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:03:13,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:14,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:03:14,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:03:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:03:18,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:23,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:28,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:29,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:03:29,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:03:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:03:34,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:39,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:03:40,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:44,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:03:44,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:03:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:03:45,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:50,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:55,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:03:59,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:03:59,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:03:59,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:04:01,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:06,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:09,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:04:12,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:14,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:04:14,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:04:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:04:17,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:22,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:27,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:29,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:04:29,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:04:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:04:33,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:38,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:39,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:04:44,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:44,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:04:44,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:04:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:04:49,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:54,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:04:59,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:04:59,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:04:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:04:59,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:04,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:09,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:05:10,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:14,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:05:14,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:05:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:05:15,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:20,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:25,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:29,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:05:29,617 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:05:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:05:31,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:36,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:39,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:05:42,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:44,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:05:44,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:05:44,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:05:47,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:52,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:57,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:05:59,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:05:59,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:05:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:06:03,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:08,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:09,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:06:14,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:14,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:06:14,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:06:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:06:19,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:24,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:29,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:06:29,620 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:06:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:06:30,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:35,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:39,690 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:06:41,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:44,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:06:44,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:06:44,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:06:46,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:51,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:56,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:06:59,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:06:59,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:06:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:07:01,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:06,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:09,691 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:07:12,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:14,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:07:14,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:07:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:07:17,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:22,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:27,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:29,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:07:29,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:07:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:07:33,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:38,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:39,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:07:44,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:44,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:07:44,637 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:07:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:07:49,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:54,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:07:59,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:07:59,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:07:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:07:59,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:04,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:09,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:08:10,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:14,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:08:14,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:08:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:08:15,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:20,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:25,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:29,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:08:29,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:08:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:08:30,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:35,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:39,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:08:41,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:44,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:08:44,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:08:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:08:46,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:51,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:56,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:08:59,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:08:59,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:08:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:09:02,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:07,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:09,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:09:13,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:14,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:09:14,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:09:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:09:18,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:23,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:28,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:29,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:09:29,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:09:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:09:34,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:39,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:09:40,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:44,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:09:44,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:09:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:09:45,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:50,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:55,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:09:59,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:09:59,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:09:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:10:00,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:05,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:09,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:10:11,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:14,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:10:14,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:10:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:10:16,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:21,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:26,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:29,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:10:29,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:10:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:10:31,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:36,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:39,695 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:10:42,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:44,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:10:44,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:10:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:10:47,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:52,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:57,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:10:59,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:10:59,633 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:10:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:11:02,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:07,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:09,695 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:11:13,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:14,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:11:14,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:11:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:11:18,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:23,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:28,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:29,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:11:29,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:11:29,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:11:33,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:38,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:39,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:11:44,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:44,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:11:44,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:11:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:11:49,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:54,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:11:59,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:11:59,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:11:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:12:00,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:05,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:09,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:12:11,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:14,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:12:14,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:12:14,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:12:16,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:21,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:26,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:12:29,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:12:29,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:12:32,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:37,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:39,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:12:43,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:44,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:12:44,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:12:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:12:48,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:53,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:58,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:12:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:12:59,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:12:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:13:02,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:13:04,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:09,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:09,698 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:13:14,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:13:14,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:13:14,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:13:14,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:19,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:24,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:13:29,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:13:29,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:13:30,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:35,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:39,700 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:13:41,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:44,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:13:44,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:13:44,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:13:46,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:51,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:56,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:13:59,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:13:59,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:13:59,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:14:01,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:06,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:09,699 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:14:12,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:14,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:14:14,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:14:14,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:14:17,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:22,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:27,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:29,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:14:29,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:14:29,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:14:32,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:37,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:39,700 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:14:43,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:44,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:14:44,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:14:44,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:14:48,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:53,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:58,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:14:59,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:14:59,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:14:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:15:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:09,700 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:15:10,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:14,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:15:14,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:15:14,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:15:15,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:20,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:25,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:29,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:15:29,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:15:29,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:15:30,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:35,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:39,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:15:41,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:44,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:15:44,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:15:44,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:15:46,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:51,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:56,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:15:59,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:15:59,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:15:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:16:02,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:07,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:09,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:16:13,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:14,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:16:14,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:16:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:16:18,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:23,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:28,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:29,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:16:29,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:16:29,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:16:30,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:16:33,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:38,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:39,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:16:44,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:16:44,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:16:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:16:44,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:49,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:54,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:16:59,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:16:59,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:16:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:16:59,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:04,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:09,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:17:10,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:14,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:17:14,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:17:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:17:15,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:20,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:25,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:17:29,635 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:17:29,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:17:31,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:36,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:39,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:17:42,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:44,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:17:44,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:17:44,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:17:47,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:52,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:57,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:17:59,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:17:59,636 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:17:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:18:03,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:08,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:09,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:18:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:18:14,636 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:18:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:18:14,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:19,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:24,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:29,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:18:29,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:18:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:18:29,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:34,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:39,705 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:18:40,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:44,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:18:44,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:18:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:18:45,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:50,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:18:59,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:18:59,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:18:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:19:00,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:05,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:09,705 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:19:11,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:14,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:19:14,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:19:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:19:16,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:26,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:29,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:19:29,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:19:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:19:32,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:37,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:39,705 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:19:43,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:44,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:19:44,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:19:44,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:19:48,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:53,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:58,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:19:59,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:19:59,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:19:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:20:03,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:08,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:09,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:20:14,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:20:14,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:20:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:20:14,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:19,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:24,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:29,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:20:29,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:20:29,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:20:30,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:35,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:39,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:20:41,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:44,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:20:44,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:20:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:20:46,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:51,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:56,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:20:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:20:59,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:20:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:21:02,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:07,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:09,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:21:13,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:14,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:21:14,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:21:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:21:18,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:23,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:28,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:29,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:21:29,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:21:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:21:34,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:39,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:21:40,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:44,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:21:44,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:21:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:21:45,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:50,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:55,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:21:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:21:59,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:21:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:22:00,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:05,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:09,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:22:11,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:14,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:22:14,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:22:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:22:16,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:21,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:26,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:29,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:22:29,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:22:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:22:31,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:36,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:39,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:22:42,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:44,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:22:44,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:22:44,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:22:47,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:52,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:57,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:22:59,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:22:59,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:22:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:23:02,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:07,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:09,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:23:13,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:14,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:23:14,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:23:14,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:23:18,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:23,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:28,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:29,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:23:29,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:23:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:23:33,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:38,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:39,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:23:44,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:23:44,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:23:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:23:44,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:49,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:54,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:23:59,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:23:59,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:23:59,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:23:59,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:04,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:09,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:24:10,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:14,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:24:14,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:24:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:24:15,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:20,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:25,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:29,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:24:29,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:24:29,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:24:31,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:36,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:39,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:24:42,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:44,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:24:44,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:24:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:24:47,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:52,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:57,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:24:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:24:59,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:24:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:25:03,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:08,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:09,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:25:14,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:25:14,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:25:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:25:14,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:19,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:24,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:29,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:25:29,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:25:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:25:30,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:35,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:39,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:25:41,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:44,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:25:44,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:25:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:25:46,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:51,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:56,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:25:59,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:25:59,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:25:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:26:02,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:07,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:09,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:26:13,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:14,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:26:14,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:26:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:26:18,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:23,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:28,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:29,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:26:29,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:26:29,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:26:34,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:39,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:26:40,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:44,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:26:44,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:26:44,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:26:45,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:50,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:26:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:26:59,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:26:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:27:01,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:07,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:09,724 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:27:09,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:27:13,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:27:14,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:27:14,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:27:18,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:23,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:28,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:29,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:27:29,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:27:29,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:27:33,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:38,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:39,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:27:44,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:27:44,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:27:44,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:27:44,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:49,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:54,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:27:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:27:59,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:27:59,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:28:00,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:05,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:09,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:28:11,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:28:14,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:28:14,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:28:16,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:21,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:26,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:29,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:28:29,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:28:29,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:28:32,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:37,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:39,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:28:43,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:44,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:28:44,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:28:44,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:28:48,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:53,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:58,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:28:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:28:59,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:28:59,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:29:03,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:08,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:09,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:29:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:29:14,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:29:14,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:29:14,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:19,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:24,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:29,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:29:29,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:29:29,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:29:29,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:34,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:39,716 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:29:40,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:44,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:29:44,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:29:44,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:29:45,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:50,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:55,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:29:59,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:29:59,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:29:59,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:30:01,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:06,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:09,717 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:30:12,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:14,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:30:14,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:30:14,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:30:17,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:22,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:27,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:29,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:30:29,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:30:29,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:30:32,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:37,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:39,717 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:30:43,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:44,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:30:44,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:30:44,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:30:48,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:53,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:58,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:30:59,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:30:59,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:30:59,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:31:03,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:08,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:09,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:31:14,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:31:14,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:31:14,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:31:14,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:19,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:24,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:29,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:31:29,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:31:29,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:31:29,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:34,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:39,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:31:40,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:44,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:31:44,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:31:44,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:31:45,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:50,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:55,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:31:59,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:31:59,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:31:59,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:32:00,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:05,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:09,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:32:11,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:14,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:32:14,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:32:14,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:32:16,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:21,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:26,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:29,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:32:29,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:32:29,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:32:31,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:36,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:39,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:32:42,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:32:44,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:32:44,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:32:47,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:52,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:57,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:32:59,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:32:59,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:32:59,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:33:03,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:08,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:09,721 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:33:14,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:33:14,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:33:14,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:33:14,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:19,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:24,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:29,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:33:29,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:33:29,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:33:29,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:34,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:39,722 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:33:40,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:33:44,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:33:44,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:33:45,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:50,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:55,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:33:59,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:33:59,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:33:59,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:34:01,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:06,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:09,722 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:34:12,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:14,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:34:14,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:34:14,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:34:17,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:22,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:27,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:34:29,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:34:29,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:34:32,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:37,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:39,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:34:43,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:34:44,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:34:44,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:34:48,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:51,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:34:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:59,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:34:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:34:59,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:34:59,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:35:04,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:09,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:35:10,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:35:14,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:35:14,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:35:15,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:20,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:25,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:35:29,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:35:29,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:35:31,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:36,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:39,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:35:42,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:44,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:35:44,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:35:44,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:35:47,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:52,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:57,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:35:59,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:35:59,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:35:59,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:36:03,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:08,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:09,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:36:14,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:36:14,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:36:14,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:36:14,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:19,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:24,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:29,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:36:29,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:36:29,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:36:30,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:35,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:39,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:36:41,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:44,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:36:44,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:36:44,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:36:46,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:51,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:56,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:36:59,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:36:59,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:36:59,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:37:01,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:06,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:09,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:37:12,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:14,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:37:14,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:37:14,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:37:17,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:22,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:27,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:29,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:37:29,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:37:29,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:37:32,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:37,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:39,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:37:43,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:44,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:37:44,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:37:44,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:37:48,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:53,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:58,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:37:59,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:37:59,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:37:59,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:38:04,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:09,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:38:10,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:14,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:38:14,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:38:14,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:38:15,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:20,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:25,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:29,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:38:29,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:38:29,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:38:31,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:36,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:39,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:38:42,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:44,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:38:44,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:38:44,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:38:47,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:52,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:57,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:38:59,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:38:59,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:38:59,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:39:03,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:08,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:09,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:39:14,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:39:14,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:39:14,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:39:14,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:19,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:24,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:29,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:39:29,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:39:29,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:39:30,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:35,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:39,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:39:41,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:44,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:39:44,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:39:44,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:39:46,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:51,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:56,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:39:59,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:39:59,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:39:59,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:40:01,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:06,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:09,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:40:12,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:14,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:40:14,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:40:14,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:40:17,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:22,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:27,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:29,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:40:29,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:40:29,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:40:33,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:38,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:39,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:40:44,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:40:44,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:40:44,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:40:44,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:49,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:54,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:40:59,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:40:59,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:40:59,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:41:00,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:05,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:09,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:41:11,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:14,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:41:14,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:41:14,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:41:16,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:21,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:27,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:28,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:41:29,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:41:29,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:41:29,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:41:32,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:37,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:39,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:41:43,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:44,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:41:44,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:41:44,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:41:48,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:53,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:58,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:41:59,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:41:59,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:41:59,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:42:03,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:08,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:09,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:42:14,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:42:14,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:42:14,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:42:14,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:19,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:24,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:29,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:42:29,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:42:29,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:42:30,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:35,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:39,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:42:41,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:44,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:42:44,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:42:44,684 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:42:46,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:51,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:56,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:42:59,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:42:59,682 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:42:59,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:43:01,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:06,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:43:07,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:09,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:43:12,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:14,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:43:14,683 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:43:14,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:43:17,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:22,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:27,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:32,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:37,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:39,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:43:43,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:48,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:53,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:43:58,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:03,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:08,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:09,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:44:13,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:23,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:28,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:30,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:44:30,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:44:30,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:44:34,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:39,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:39,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:44:40,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:44:44,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:45,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:44:45,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:44:45,047 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:44:50,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:55,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:44:58,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:45:00,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:45:00,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:45:00,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:45:01,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:06,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:09,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:45:09,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:45:11,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:45:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:45:15,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:45:17,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:19,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:45:22,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:27,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:30,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:45:30,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:45:30,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:45:31,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:45:33,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:38,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:39,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:45:42,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:45:44,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:45,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:45:45,047 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:45:45,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:45:49,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:52,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:45:54,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:45:59,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:00,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:46:00,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:46:00,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:46:02,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:46:04,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:09,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:09,746 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:46:12,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:46:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:46:15,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:46:15,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:46:15,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:20,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:22,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:46:25,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:30,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:31,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:46:31,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:46:31,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:46:34,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:46:35,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:39,748 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:46:40,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:44,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:46:45,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:46,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:46:46,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:46:46,551 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:46:51,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:46:54,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:46:57,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:02,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:47:02,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:47:02,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:47:03,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:04,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:47:08,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:09,747 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:47:14,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:14,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:47:17,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:47:17,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:47:17,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:47:19,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:24,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:24,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:47:29,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:34,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:47:34,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:47:34,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:47:35,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:36,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:47:39,748 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:47:40,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:46,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:46,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:47:49,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:47:49,121 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:47:49,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:47:51,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:56,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:47:56,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:01,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:04,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:48:04,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:48:04,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:48:05,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 12:48:05,646 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 12:48:05,650 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 12:48:05,651 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 12:48:05,651 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 12:48:05,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 12:48:05,666 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 12:48:05,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:05,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_created():271] file/dir created: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 12:48:07,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:08,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:09,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:48:12,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:18,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:18,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:19,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/config.yaml -2023-11-19 12:48:19,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:48:19,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:48:19,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:48:24,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:28,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:30,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:34,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:48:34,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:48:34,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:48:35,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:38,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:39,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:48:40,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:45,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:49,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:48:49,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:48:49,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:48:50,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:48:51,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:48:56,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:00,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:49:01,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:04,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:49:04,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:49:04,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:49:07,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:09,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:49:12,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:49:13,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:18,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:19,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:49:19,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:49:19,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:49:23,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:28,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:49:29,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:34,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:34,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:49:34,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:49:34,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:49:36,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:49:39,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:49:39,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:44,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:46,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:49:49,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:49,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:49:49,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:49:49,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:49:55,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:49:56,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:50:00,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:04,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:50:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:50:04,793 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:50:06,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:06,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:50:09,753 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:50:11,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:17,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:50:17,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:19,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:50:19,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:50:19,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:50:23,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:28,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:29,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:50:33,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:34,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:50:34,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:50:34,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:50:38,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:50:39,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:39,753 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:50:44,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:48,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:50:49,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:50:49,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:49,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:50:49,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:50:54,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:50:56,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:00,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:51:04,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:51:04,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:51:05,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:06,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:09,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:51:11,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:16,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:17,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:19,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:51:19,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:51:19,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:51:22,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:26,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:28,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:33,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:34,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 12:51:34,745 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 12:51:34,746 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 12:51:34,748 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 12:51:34,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:51:34,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:51:34,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:51:34,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 12:51:36,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:39,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:39,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:51:44,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:47,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:49,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:51:49,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:51:49,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:51:49,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:51:55,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:51:55,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:00,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:04,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:52:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:52:04,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:52:05,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:52:05,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:09,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:52:11,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:15,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:52:16,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:19,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:52:19,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:52:19,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:52:21,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:25,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:52:27,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:32,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:34,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:52:34,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:52:34,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:52:34,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:52:37,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:39,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:52:43,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:44,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:52:48,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:49,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:52:49,793 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:52:49,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:52:54,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:52:54,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:52:59,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:04,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:04,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:53:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:53:04,793 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:53:04,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:53:09,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:53:09,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:14,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:53:15,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:19,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:53:19,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:53:19,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:53:21,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:24,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:53:26,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:31,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:32,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:53:34,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:53:34,793 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:53:34,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:53:37,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:39,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:53:42,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:53:43,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:48,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:49,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:53:49,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:53:49,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:53:53,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:53:55,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:53:58,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:04,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:04,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:54:04,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:54:04,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:54:05,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:09,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:54:09,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:13,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:15,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:20,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:20,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:54:20,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:54:20,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:54:23,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:25,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:30,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:33,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:35,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:54:35,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:54:35,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:54:35,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:39,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:54:41,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:43,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:46,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:50,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 12:54:50,122 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 12:54:50,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 12:54:50,127 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 12:54:50,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:54:50,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:54:50,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:54:50,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 12:54:51,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:51,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:54:52,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:54:56,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:01,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:55:01,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:05,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:55:05,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:55:05,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:55:07,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:09,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:55:10,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:55:12,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:17,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:20,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:55:20,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:55:20,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:55:20,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:55:23,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:28,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:28,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:55:33,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:36,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:55:36,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:55:36,228 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:55:38,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:55:39,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:39,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:55:44,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:48,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:55:49,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:51,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:55:51,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:55:51,226 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:55:55,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:55:59,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:00,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:05,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:06,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:56:06,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:56:06,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:56:09,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:09,772 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:56:11,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:16,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:19,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:21,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:56:21,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:56:21,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:21,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:56:27,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:29,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:32,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:37,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:56:37,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:56:37,112 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:56:38,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:39,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:39,772 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:56:43,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:49,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:49,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:52,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:56:52,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:56:52,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:56:54,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:56:59,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:56:59,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:04,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:07,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:57:07,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:57:07,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:57:09,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:57:09,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:57:10,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:15,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:18,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:57:21,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:22,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:57:22,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:57:22,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:57:27,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:28,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:57:32,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:36,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:57:37,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:57:37,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:57:37,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:57:38,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:39,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:57:43,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:46,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:57:48,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:52,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:57:52,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:57:52,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:57:54,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:57:57,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:57:59,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:04,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:04,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 12:58:04,865 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 12:58:04,866 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 12:58:04,869 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 12:58:05,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 12:58:07,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:58:07,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:58:07,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:58:07,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:58:09,774 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:58:09,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:15,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:17,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:58:20,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:22,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:58:22,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:58:22,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:58:25,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:58:25,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:30,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:35,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:58:35,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:37,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:58:37,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:58:37,819 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:58:39,774 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:58:41,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:45,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:58:47,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:52,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:58:52,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:58:52,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:58:52,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:58:55,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:58:58,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:03,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:05,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:59:07,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:59:07,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:59:07,819 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:59:09,776 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:59:09,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:15,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:59:15,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:20,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:22,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:59:22,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:59:22,819 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:59:25,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:59:25,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:30,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:33,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:59:36,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:37,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:59:37,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:59:37,820 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:59:39,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 12:59:41,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:45,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:59:46,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:51,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 12:59:52,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 12:59:52,820 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 12:59:52,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 12:59:54,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 12:59:57,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:02,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:05,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:00:07,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:00:07,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:00:07,820 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:00:08,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:09,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:00:13,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:15,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:00:18,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:22,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:00:22,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:00:22,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:00:23,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:00:24,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:29,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:33,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:00:34,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:37,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:00:37,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:00:37,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:00:39,783 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:00:39,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:45,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:00:45,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:50,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:00:52,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:00:52,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:00:52,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:00:55,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:00:55,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:00,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:05,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:01:06,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:07,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:01:07,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:01:07,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:01:09,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:01:12,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:15,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:01:17,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:22,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:01:22,832 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:01:22,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:01:23,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:23,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:01:23,667 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:01:23,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:01:23,670 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:01:24,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:01:24,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:01:25,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:01:28,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:33,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:35,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:01:37,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:01:37,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:01:37,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:01:39,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:39,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:01:44,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:45,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:01:49,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:53,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:01:53,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:01:53,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:01:55,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:01:55,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:02:00,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:05,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:06,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:02:08,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:02:08,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:02:08,072 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:02:09,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:02:10,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:15,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:20,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:23,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:02:23,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:02:23,072 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:02:26,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:31,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:36,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:38,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:02:38,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:02:38,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:02:39,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:02:41,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:45,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:02:47,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:52,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:02:53,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:02:53,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:02:53,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:02:55,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:02:58,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:03,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:05,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:03:08,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:03:08,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:03:08,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:03:08,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:09,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:03:13,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:15,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:03:18,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:23,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:03:23,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:03:23,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:03:24,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:25,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:03:29,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:35,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:35,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:03:38,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:03:38,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:03:38,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:03:39,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:03:40,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:45,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:03:45,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:50,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:03:53,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:03:53,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:03:53,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:03:53,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:03:56,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:01,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:03,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:04:06,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:08,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:04:08,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:04:08,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:04:09,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:04:11,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:13,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:04:17,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:22,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:23,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:04:23,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:04:23,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:04:27,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:28,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:04:32,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:38,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:04:38,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:04:38,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:38,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:04:38,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:04:39,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:04:43,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:48,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:04:49,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:53,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:04:53,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:04:53,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:04:55,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:04:58,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:05:01,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:06,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:08,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:05:08,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:05:08,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:05:09,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:05:09,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:05:11,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:16,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:17,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:05:17,166 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:05:17,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:05:17,168 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:05:17,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:05:19,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:05:22,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:23,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:05:23,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:05:23,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:05:27,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:05:27,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:32,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:37,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:05:38,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:05:38,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:05:38,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:38,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:05:39,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:05:43,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:47,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:05:49,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:54,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:05:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:05:54,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:05:55,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:05:57,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:00,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:05,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:05,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:09,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:06:09,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:06:09,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:06:09,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:06:10,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:15,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:16,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:21,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:24,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:06:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:06:24,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:06:25,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:27,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:32,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:35,962 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:37,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:39,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:06:39,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:06:39,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:06:39,793 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:06:42,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:46,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:48,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:54,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:06:54,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:06:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:06:54,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:06:56,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:06:59,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:04,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:06,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:07:09,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:07:09,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:07:09,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:07:09,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:09,793 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:07:15,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:16,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:07:20,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:24,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:07:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:07:24,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:07:26,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:07:26,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:31,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:36,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:07:36,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:39,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:07:39,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:07:39,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:07:39,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:07:42,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:46,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:07:48,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:53,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:07:54,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:07:54,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:07:54,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:07:55,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:07:59,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:03,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:08:05,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:09,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:08:09,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:08:09,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:08:09,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:08:10,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:13,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:08:16,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:21,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:23,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:08:24,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:08:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:08:24,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:08:26,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:31,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:31,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:08:31,806 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:08:31,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:08:31,809 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:08:31,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:08:33,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:08:36,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:39,796 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:08:41,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:08:41,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:08:41,053 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:08:42,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:43,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:08:47,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:52,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:08:54,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:08:56,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:08:56,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:08:56,050 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:08:57,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:02,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:04,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:09:07,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:09,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:09:11,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:09:11,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:09:11,052 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:09:13,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:14,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:09:18,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:23,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:24,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:09:26,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:09:26,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:09:26,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:09:28,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:33,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:34,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:09:38,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:39,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:09:42,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:09:42,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:09:42,419 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:09:44,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:46,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:09:49,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:54,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:09:55,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:09:57,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:09:57,419 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:09:57,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:10:00,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:04,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:10:06,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:09,799 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:10:11,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:12,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:10:12,420 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:10:12,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:10:13,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:10:17,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:23,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:23,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:10:27,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:10:27,419 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:10:27,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:10:28,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:33,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:34,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:10:38,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:39,800 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:10:42,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:10:42,419 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:10:42,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:10:44,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:10:44,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:49,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:54,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:10:54,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:10:57,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:10:57,420 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:10:57,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:11:00,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:04,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:05,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:09,801 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:11:10,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:12,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:12,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:11:12,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:11:12,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:11:16,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:21,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:22,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:26,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:27,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:11:27,420 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:11:27,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:11:31,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:34,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:36,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:39,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:11:42,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:11:42,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:42,420 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:11:42,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:11:44,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:47,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:51,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:11:51,877 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:11:51,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:11:51,879 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:11:52,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:11:52,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:11:53,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:54,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:11:57,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:11:57,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:11:57,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:11:58,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:02,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:12:04,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:09,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:09,804 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:12:12,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:12:12,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:12:12,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:12:12,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:12:14,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:19,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:22,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:12:24,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:27,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:12:27,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:12:27,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:12:30,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:31,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:12:35,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:39,805 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:12:40,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:42,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:12:42,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:12:42,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:12:42,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:12:46,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:52,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:12:52,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:57,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:12:57,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:12:57,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:12:57,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:13:02,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:13:02,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:07,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:09,807 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:13:12,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:13:12,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:13:12,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:12,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:13:12,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:13:17,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:22,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:13:23,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:27,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:13:27,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:13:28,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:29,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:13:32,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:13:34,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:39,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:39,808 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:13:42,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:13:42,422 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:13:42,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:13:44,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:13:44,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:49,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:52,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:13:55,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:13:57,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:13:57,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:13:59,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:14:00,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:02,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:14:05,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:09,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:14:11,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:12,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:14:12,423 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:14:12,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:14:14,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:14:16,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:22,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:22,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:14:27,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:14:27,423 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:14:27,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:30,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:14:32,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:14:33,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:38,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:39,812 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:14:42,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:14:42,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:14:42,423 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:14:43,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:45,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:14:48,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:54,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:14:54,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:14:57,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:14:57,423 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:15:00,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:00,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:15:04,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:15:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:09,813 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:15:10,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:11,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:15:11,695 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:15:11,696 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:15:11,699 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:15:12,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:15:12,423 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:15:12,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:15:14,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:15:15,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:15:16,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:21,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:24,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:15:26,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:27,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:15:27,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:15:31,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:15:32,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:34,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:15:37,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:39,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:15:42,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:15:42,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:15:42,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:44,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:15:46,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:15:48,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:53,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:15:54,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:15:57,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:15:57,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:15:58,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:01,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:16:01,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:02,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:04,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:07,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:08,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:09,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:16:09,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:12,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:16:12,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:16:15,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:16,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:16:19,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:21,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:26,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:27,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:16:27,424 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:16:31,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:31,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:16:31,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:36,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:38,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:39,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:16:41,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:42,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:16:42,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:16:46,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:16:47,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:48,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:16:52,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:57,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:16:57,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:16:57,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:16:58,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:17:01,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:17:02,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:07,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:09,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:17:10,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:17:12,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:17:12,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:17:13,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:16,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:17:18,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:17:19,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:24,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:27,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:17:27,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:17:28,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:17:29,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:31,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:17:34,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:38,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:17:39,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:17:39,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:42,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:17:42,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:17:45,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:46,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:17:48,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:17:51,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:56,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:17:57,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:17:57,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:17:59,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:01,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:18:01,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:06,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:09,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:09,817 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:18:11,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:12,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:18:12,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:18:16,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:18:17,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:19,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:22,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:27,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:18:27,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:18:27,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:27,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:18:27,707 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:18:27,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:18:27,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:18:28,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:18:28,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:29,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:31,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:18:32,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:38,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:39,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:39,818 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:18:42,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:18:42,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:18:44,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:46,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:18:49,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:18:50,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:55,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:18:57,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:18:57,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:19:00,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:01,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:19:02,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:19:05,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:09,820 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:19:10,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:12,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:19:12,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:19:12,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:19:16,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:19:16,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:21,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:22,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:19:26,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:27,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:19:27,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:19:31,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:19:32,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:19:32,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:37,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:39,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:19:42,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:19:42,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:19:42,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:19:43,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:46,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:19:48,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:52,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:19:54,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:19:57,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:19:57,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:19:59,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:01,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:20:02,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:20:05,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:09,822 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:20:11,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:12,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:20:12,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:20:13,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:20:16,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:20:16,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:22,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:23,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:20:27,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:28,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:20:28,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:20:31,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:20:31,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:20:32,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:37,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:39,822 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:20:41,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:20:42,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:43,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:20:43,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:20:46,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:20:47,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:51,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:20:52,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:57,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:20:58,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:20:58,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:21:01,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:21:01,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:21:03,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:08,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:09,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:21:11,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:21:13,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:21:13,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:21:13,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:17,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:21:19,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:21,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:21:24,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:28,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:21:28,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:21:29,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:30,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:21:32,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:21:34,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:39,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:21:40,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:21:40,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:43,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:21:43,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:21:45,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:47,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:21:48,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:21:48,316 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:21:48,316 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:21:48,318 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:21:48,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:21:50,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:21:51,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:56,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:21:58,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:21:58,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:21:58,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:22:01,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:02,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:22:06,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:08,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:22:09,824 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:22:12,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:13,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:22:13,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:22:17,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:22:18,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:19,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:22:23,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:28,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:28,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:22:28,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:22:29,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:22:32,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:22:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:39,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:22:39,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:39,824 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:22:44,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:45,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:22:45,169 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:22:47,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:22:49,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:22:50,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:55,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:22:57,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:23:00,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:23:00,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:23:01,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:02,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:23:06,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:07,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:23:09,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:23:11,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:15,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:23:15,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:23:17,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:17,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:23:22,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:27,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:30,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:23:30,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:23:33,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:33,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:23:37,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:23:38,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:39,826 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:23:43,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:45,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:23:45,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:23:47,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:23:48,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:23:49,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:54,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:23:57,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:23:59,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:00,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:24:00,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:24:03,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:24:04,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:05,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:24:09,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:09,826 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:24:14,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:24:14,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:15,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:24:15,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:24:18,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:24:20,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:24,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:24:25,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:30,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:24:30,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:24:31,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:33,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:24:35,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:24:36,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:39,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:24:41,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:45,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:24:45,169 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:24:45,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:24:47,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:48,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:24:52,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:24:55,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:24:57,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:01,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:25:01,556 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:25:03,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:25:03,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:05,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:08,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:09,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:25:13,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:14,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:16,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:25:16,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:25:18,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:25:19,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:21,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:25:21,886 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:25:21,887 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:25:21,890 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:25:22,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:25:22,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:23,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:24,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:29,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:31,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:25:31,558 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:25:33,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:25:33,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:35,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:39,831 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:25:40,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:43,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:46,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:25:46,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:46,556 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:25:48,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:25:52,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:25:53,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:25:57,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:01,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:26:01,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:26:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:02,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:26:03,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:26:07,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:09,832 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:26:11,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:26:13,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:16,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:26:16,556 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:26:18,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:19,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:26:21,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:26:24,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:29,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:31,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:26:31,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:26:31,556 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:26:34,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:26:34,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:39,832 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:26:39,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:41,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:26:44,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:26:47,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:26:49,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:26:49,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:26:50,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:55,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:26:59,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:27:01,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:02,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:27:02,338 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:27:04,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:27:06,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:09,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:27:09,833 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:27:11,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:16,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:17,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:27:17,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:27:19,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:27:19,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:27:22,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:28,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:29,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:27:32,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:27:32,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:27:33,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:34,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:27:38,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:39,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:27:39,833 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:27:43,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:27:47,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:27:49,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:27:49,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:49,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:27:54,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:27:59,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:00,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:02,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:28:02,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:28:05,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:05,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:28:07,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:09,834 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:28:10,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:15,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:17,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:28:17,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:28:17,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:20,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:28:21,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:27,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:28,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:32,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:32,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:28:32,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:28:35,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:28:36,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:28:36,327 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:28:36,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:28:36,329 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:28:37,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:28:37,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:37,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:39,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:39,834 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:28:42,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:47,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:28:47,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:28:48,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:50,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:28:53,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:28:57,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:28:59,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:03,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:29:03,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:29:04,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:05,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:29:07,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:29:09,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:29:10,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:15,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:29:16,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:18,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:29:18,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:29:20,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:29:22,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:25,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:29:27,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:32,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:33,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:29:33,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:29:35,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:29:35,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:29:38,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:39,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:29:43,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:45,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:29:48,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:48,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:29:48,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:29:51,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:29:53,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:29:53,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:29:58,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:03,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:30:03,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:30:03,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:30:04,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:06,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:30:09,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:09,837 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:30:14,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:30:14,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:18,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:30:18,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:30:20,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:21,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:30:24,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:30:25,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:30,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:33,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:30:33,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:30:34,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:30:36,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:36,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:30:39,838 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:30:41,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:44,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:30:47,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:48,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:30:48,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:30:51,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:30:52,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:30:53,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:30:57,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:03,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:03,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:03,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:31:03,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:31:06,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:31:09,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:09,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:31:11,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:14,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:19,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:31:19,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:31:19,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:21,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:21,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:31:25,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:31,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:31,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:34,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:31:34,621 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:31:36,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:31:36,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:39,839 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:31:41,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:42,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:47,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:49,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:31:49,042 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:31:49,042 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:31:49,044 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:31:49,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:31:49,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:31:49,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:31:49,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:51,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:31:51,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:31:53,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:58,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:31:59,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:32:03,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:04,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:32:04,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:32:07,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:32:08,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:09,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:32:09,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:32:14,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:19,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:32:19,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:32:19,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:32:20,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:22,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:32:25,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:30,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:32:31,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:34,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:32:34,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:32:36,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:37,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:32:39,841 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:32:40,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:32:41,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:46,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:49,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:32:49,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:32:50,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:32:51,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:32:52,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:32:56,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:00,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:33:02,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:05,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:33:05,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:33:07,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:33:07,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:08,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:33:09,841 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:33:13,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:18,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:33:19,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:20,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:33:20,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:33:22,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:33:25,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:28,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:33:30,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:35,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:35,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:33:35,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:33:37,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:33:37,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:33:39,842 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:33:40,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:45,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:50,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:33:50,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:33:50,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:33:51,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:33:52,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:33:56,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:01,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:34:01,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:05,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:34:05,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:34:07,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:07,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:34:09,845 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:34:11,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:34:12,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:17,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:20,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:34:20,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:34:21,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:34:22,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:34:23,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:28,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:31,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:34:33,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:35,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:34:35,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:34:37,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:34:39,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:39,844 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:34:44,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:49,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:34:50,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:34:50,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:34:50,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:34:52,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:34:55,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:00,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:00,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:05,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:05,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:35:05,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:35:08,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:35:09,845 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:35:10,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:10,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:15,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:18,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:35:18,894 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:35:18,895 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:35:18,897 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:35:19,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:35:20,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:20,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:20,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:35:20,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:35:21,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:23,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:35:26,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:30,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:31,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:36,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:37,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:35:37,360 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:35:38,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:35:39,845 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:35:40,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:41,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:46,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:50,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:35:51,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:52,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:35:52,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:35:53,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:35:57,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:35:58,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:36:02,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:07,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:36:07,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:36:07,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:08,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:36:08,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:36:09,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:36:12,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:17,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:36:17,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:22,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:36:22,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:36:23,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:36:23,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:27,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:36:28,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:33,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:37,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:36:37,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:36:37,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:36:38,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:36:39,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:39,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:36:44,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:47,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:36:50,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:52,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:36:52,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:36:53,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:36:56,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:36:58,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:37:01,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:07,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:07,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:37:07,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:37:08,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:37:08,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:37:09,847 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:37:12,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:18,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:37:18,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:22,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:37:22,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:37:24,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:37:24,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:28,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:37:29,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:34,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:36,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:37:37,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:37:37,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:37:39,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:37:39,847 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:37:39,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:44,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:46,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:37:50,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:53,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:37:53,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:37:54,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:37:55,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:37:56,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:38:00,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:06,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:38:06,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:08,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:38:08,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:38:09,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:38:09,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:38:11,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:16,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:38:17,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:22,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:23,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:38:23,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:38:24,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:38:26,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:38:28,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:33,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:33,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:38:33,406 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:38:33,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:38:33,408 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:38:33,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:38:34,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:38:35,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:38:38,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:38,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:38:38,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:38:39,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:38:39,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:38:43,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:48,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:53,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:38:53,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:38:53,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:38:54,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:38:58,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:03,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:05,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:39:08,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:39:08,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:39:09,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:39:09,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:39:09,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:15,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:15,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:39:20,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:23,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:39:23,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:39:24,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:39:25,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:26,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:39:30,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:36,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:39:36,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:38,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:39:38,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:39:39,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:39:39,850 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:39:41,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:46,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:39:47,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:52,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:39:53,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:39:53,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:39:54,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:39:56,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:39:58,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:03,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:04,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:40:08,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:08,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:40:08,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:40:09,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:40:09,850 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:40:14,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:14,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:40:19,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:23,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:40:23,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:40:24,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:40:24,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:40:25,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:30,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:34,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:40:36,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:38,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:40:38,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:40:39,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:40:39,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:40:41,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:44,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:40:46,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:51,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:40:53,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:40:53,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:40:54,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:40:54,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:40:57,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:02,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:04,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:41:07,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:08,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:41:08,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:41:09,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:41:09,852 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:41:12,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:14,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:41:18,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:23,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:41:23,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:23,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:41:24,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:41:24,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:41:28,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:34,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:34,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:41:38,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:41:38,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:41:39,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:41:39,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:41,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:41:45,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:41:45,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:50,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:53,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:41:53,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:41:54,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:41:55,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:41:56,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:42:00,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:02,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:42:06,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:08,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:42:08,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:42:09,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:42:11,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:42:12,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:12,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:42:12,339 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:42:12,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:42:12,342 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:42:13,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:42:14,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:42:17,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:22,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:42:22,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:23,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:42:23,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:42:26,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:42:28,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:32,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:42:33,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:38,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:39,870 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:42:40,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:42:40,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:42:41,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:42:42,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:42:44,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:49,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:52,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:42:54,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:42:55,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:42:55,188 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:42:56,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:43:00,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:02,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:43:05,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:09,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:43:10,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:43:10,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:43:11,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:43:11,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:12,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:43:16,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:22,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:22,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:43:25,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:43:25,188 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:43:26,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:43:27,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:30,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:43:33,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:38,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:39,858 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:43:40,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:43:40,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:43:41,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:43:43,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:43:43,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:48,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:53,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:43:53,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:43:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:43:55,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:43:56,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:43:59,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:03,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:44:04,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:09,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:44:10,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:44:10,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:44:10,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:11,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:44:12,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:44:15,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:20,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:22,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:44:25,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:44:25,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:44:26,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:44:26,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:31,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:32,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:44:37,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:39,861 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:44:41,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:44:41,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:44:41,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:44:42,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:44,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:44:47,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:52,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:44:54,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:44:56,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:44:56,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:44:56,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:44:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:03,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:04,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:45:08,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:09,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:45:11,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:45:11,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:45:11,404 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:45:14,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:16,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:45:19,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:24,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:26,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:45:26,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:45:26,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:45:27,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:45:29,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:34,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:35,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:45:35,614 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:45:35,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:45:35,616 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:45:36,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:45:36,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:45:37,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:45:39,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:39,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:45:41,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:45:41,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:45:41,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:45:45,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:47,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:45:50,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:55,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:45:56,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:45:56,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:45:56,400 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:45:57,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:46:00,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:05,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:08,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:46:09,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:46:10,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:11,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:46:11,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:46:11,400 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:46:16,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:18,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:46:22,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:26,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:46:26,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:46:26,551 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:46:27,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:28,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:46:32,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:37,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:39,863 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:46:40,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:46:41,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:46:41,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:46:41,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:46:43,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:48,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:50,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:46:54,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:46:56,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:46:56,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:46:56,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:46:59,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:00,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:47:05,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:09,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:47:10,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:11,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:47:11,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:47:11,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:47:13,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:47:16,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:22,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:23,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:47:26,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:47:26,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:47:26,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:47:27,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:33,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:47:33,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:38,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:39,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:47:41,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:47:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:47:41,551 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:47:43,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:45,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:47:48,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:54,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:47:55,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:47:56,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:47:56,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:47:56,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:47:59,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:05,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:05,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:48:09,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:48:10,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:11,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:48:11,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:48:11,559 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:48:16,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:17,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:48:21,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:26,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:26,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:48:26,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:48:26,553 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:48:28,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:48:32,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:37,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:38,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:48:39,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:48:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:48:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:48:41,551 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:48:42,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:47,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:48,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:48:52,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:56,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:48:56,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:48:56,925 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:48:58,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:48:59,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:03,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:08,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:09,869 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:49:11,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:49:11,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:49:11,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:49:12,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:49:12,517 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:49:12,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:49:12,519 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:49:13,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:49:13,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:14,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:15,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:18,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:23,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:25,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:26,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:49:26,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:49:26,923 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:49:29,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:34,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:35,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:39,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:39,871 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:49:41,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:49:41,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:49:41,929 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:49:45,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:45,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:50,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:55,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:49:56,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:49:56,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:49:56,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:49:56,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:50:02,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:05,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:50:07,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:09,872 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:50:11,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:50:11,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:50:11,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:50:13,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:15,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:50:18,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:24,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:25,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:50:26,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:50:26,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:50:26,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:50:30,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:35,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:35,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:50:39,873 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:50:40,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:42,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:50:42,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:50:42,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:50:44,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:50:46,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:51,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:54,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:50:57,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:50:57,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:50:57,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:50:57,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:51:02,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:06,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:51:07,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:09,875 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:51:12,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:51:12,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:51:12,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:51:13,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:17,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:51:19,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:24,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:27,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:51:27,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:51:27,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:51:27,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:51:29,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:34,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:39,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:51:39,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:39,887 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:51:42,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:51:42,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:51:42,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:51:45,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:49,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:51:50,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:55,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:51:57,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:51:57,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:51:57,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:51:59,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:01,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:06,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:09,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:09,876 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:52:11,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:12,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:52:12,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:52:12,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:52:17,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:19,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:22,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:27,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:52:27,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:52:27,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:52:27,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:29,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:32,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:37,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:38,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:52:38,035 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:52:38,036 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:52:38,038 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:52:38,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:52:39,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:39,877 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:52:40,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:42,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:52:42,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:52:42,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:52:43,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:48,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:49,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:52:53,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:57,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:52:57,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:52:57,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:52:59,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:52:59,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:53:04,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:09,877 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:53:09,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:09,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:53:12,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:53:12,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:53:12,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:53:15,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:21,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:21,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:53:26,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:28,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:53:28,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:53:28,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:53:31,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:53:31,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:36,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:39,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:53:41,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:53:41,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:53:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:53:43,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:53:47,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:53,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:53:53,325 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:53:58,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:00,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:54:00,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:54:00,038 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:54:01,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:54:04,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:09,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:09,882 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:54:13,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:54:14,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:15,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:54:15,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:54:15,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:54:19,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:24,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:25,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:54:29,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:30,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:54:30,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:54:30,044 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:54:35,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:35,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:54:39,882 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:54:40,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:45,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:54:45,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:54:45,039 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:54:45,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:54:46,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:51,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:57,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:54:58,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:55:00,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:55:00,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:55:00,039 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:55:03,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:07,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:55:08,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:09,882 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:55:13,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:15,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:55:15,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:55:15,042 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:55:17,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:55:19,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:24,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:29,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:55:29,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:30,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:55:30,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:55:30,041 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:55:35,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:39,883 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:55:40,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:41,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:55:45,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:55:45,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:55:45,041 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:55:46,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:51,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:55:53,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:55:56,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:00,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:56:00,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:56:00,810 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:56:02,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:03,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:56:07,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:09,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:56:12,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:12,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:56:12,989 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:56:12,990 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:56:12,992 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:56:13,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:56:15,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:56:15,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:56:15,802 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:56:15,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:56:16,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:56:18,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:23,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:26,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:56:28,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:30,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:56:30,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:56:30,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:56:34,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:37,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:56:39,886 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:56:39,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:44,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:45,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:56:45,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:56:45,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:56:47,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:56:50,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:56:55,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:00,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:00,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:57:00,803 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:57:00,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:57:01,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:57:05,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:09,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:57:11,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:57:11,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:15,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:57:15,803 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:57:15,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:57:16,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:21,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:57:22,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:27,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:30,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:57:30,803 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:57:30,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:57:31,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:57:33,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:38,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:39,892 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:57:41,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:57:43,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:45,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:57:45,803 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:57:45,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:57:48,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:57:52,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:57:54,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:00,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:00,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:58:00,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:58:00,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:58:02,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:58:05,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:09,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:58:11,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:12,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:58:15,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:58:15,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:58:15,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:58:17,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:22,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:22,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:58:27,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:30,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:58:30,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:58:30,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:58:31,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:58:32,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:37,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:39,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:58:41,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:58:43,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:45,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:58:45,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:58:45,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:58:49,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:58:51,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:58:54,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:00,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:00,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:59:00,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:59:00,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:59:01,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:06,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:09,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:59:11,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:11,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:15,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:59:15,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:59:15,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:59:17,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:21,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:22,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:27,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:30,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:59:30,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:59:30,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:59:32,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:33,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:38,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:39,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 13:59:41,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 13:59:41,150 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 13:59:41,151 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 13:59:41,153 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 13:59:42,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 13:59:42,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:43,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:43,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:45,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 13:59:45,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 13:59:45,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 13:59:48,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:52,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 13:59:54,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 13:59:59,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:00,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:00:00,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:00:00,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:00:04,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:00:04,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:09,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:09,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:00:14,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:00:14,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:15,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:00:15,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:00:15,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:00:20,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:24,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:00:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:30,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:31,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:00:31,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:00:31,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:00:34,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:00:35,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:39,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:00:40,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:46,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:46,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:00:46,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:00:46,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:00:46,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:00:51,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:00:55,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:00:56,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:01,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:02,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:01:02,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:01:02,933 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:01:05,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:01:07,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:09,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:01:12,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:15,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:01:17,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:01:17,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:01:17,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:01:18,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:23,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:25,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:01:28,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:32,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:01:32,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:01:32,935 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:01:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:36,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:01:39,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:39,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:01:45,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:46,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:01:47,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:01:47,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:01:47,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:01:50,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:56,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:01:56,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:02:01,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:04,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:02:04,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:02:04,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:02:06,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:08,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:02:09,901 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:02:11,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:17,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:18,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:02:19,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:02:19,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:02:19,470 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:02:22,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:27,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:28,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:02:32,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:34,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:02:34,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:02:34,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:02:38,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:38,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:02:39,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:02:43,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:46,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:02:49,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:49,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:02:49,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:02:49,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:02:54,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:02:56,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:00,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:04,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:03:04,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:03:04,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:03:05,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:03:05,197 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:03:05,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:03:05,199 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:03:05,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:03:06,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:06,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:07,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:09,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:03:11,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:15,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:17,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:19,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:03:19,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:03:19,470 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:03:23,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:25,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:29,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:34,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:03:34,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:03:34,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:03:34,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:35,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:39,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:39,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:03:44,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:46,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:03:49,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:03:49,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:03:49,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:03:50,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:56,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:03:58,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:04:01,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:04,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:04:04,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:04:04,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:04:06,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:08,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:04:09,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:04:11,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:16,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:19,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:04:19,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:04:19,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:04:22,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:22,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:04:27,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:32,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:32,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:04:34,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:04:34,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:04:34,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:04:37,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:39,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:04:42,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:04:43,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:48,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:50,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:04:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:04:50,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:04:53,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:04:54,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:04:58,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:04,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:05:04,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:05,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:05:05,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:05:05,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:05:09,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:09,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:05:14,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:05:15,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:05:20,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:05:20,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:05:20,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:24,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:05:25,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:30,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:34,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:05:35,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:05:35,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:05:35,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:05:36,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:39,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:05:41,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:44,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:05:47,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:50,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:05:50,817 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:05:50,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:05:52,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:05:54,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:05:57,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:02,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:04,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:05,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:06:05,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:06:05,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:06:07,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:09,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:06:12,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:13,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:18,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:20,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:06:20,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:06:20,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:06:22,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:24,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:29,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:30,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:06:30,954 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:06:30,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:06:30,957 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:06:31,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:06:31,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:32,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:34,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:35,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:06:35,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:06:35,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:06:39,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:06:40,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:42,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:46,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:50,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:06:50,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:06:50,814 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:06:52,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:06:52,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:06:57,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:02,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:07:03,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:05,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:07:05,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:07:05,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:07:09,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:09,912 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:07:12,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:07:14,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:19,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:20,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:07:20,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:07:20,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:07:24,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:24,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:07:29,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:34,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:35,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:07:35,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:07:35,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:07:39,915 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:07:39,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:43,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:07:45,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:50,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:07:50,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:07:50,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:07:51,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:07:53,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:07:56,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:01,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:03,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:08:05,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:08:05,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:08:05,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:08:06,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:09,915 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:08:12,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:08:12,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:17,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:20,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:08:20,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:08:20,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:08:22,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:08:22,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:27,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:32,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:08:33,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:35,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:08:35,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:08:35,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:08:39,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:39,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:08:42,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:08:44,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:49,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:08:50,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:08:50,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:08:50,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:08:52,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:08:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:00,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:02,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:09:05,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:05,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:09:05,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:09:05,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:09:09,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:09:10,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:12,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:09:16,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:20,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:09:20,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:09:20,825 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:09:22,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:24,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:09:27,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:32,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:34,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:09:35,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:09:35,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:09:35,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:09:38,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:39,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:09:43,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:46,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:09:49,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:50,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:09:50,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:09:50,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:09:54,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:09:57,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:09:59,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:04,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:04,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:10:04,829 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:10:04,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:10:04,833 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:10:05,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:10:05,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:10:05,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:10:05,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:10:06,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:10:07,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:10:09,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:10:09,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:15,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:17,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:10:20,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:20,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:10:20,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:10:20,824 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:10:26,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:27,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:10:31,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:35,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:10:35,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:10:35,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:10:36,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:38,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:10:39,946 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:10:41,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:47,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:48,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:10:50,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:10:50,825 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:10:50,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:10:53,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:58,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:10:58,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:11:03,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:06,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:11:06,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:11:06,737 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:11:09,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:09,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:11:10,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:11:14,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:20,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:20,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:11:21,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:11:21,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:11:21,733 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:11:25,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:30,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:11:31,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:36,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:38,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:11:38,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:11:38,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:11:39,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:11:40,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:11:42,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:47,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:53,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:11:53,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:11:53,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:11:53,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:11:53,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:11:58,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:03,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:12:03,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:08,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:09,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:12:09,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:12:09,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:12:09,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:12:13,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:12:14,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:19,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:23,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:12:24,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:12:24,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:12:24,311 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:12:25,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:31,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:33,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:12:36,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:39,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:12:39,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:12:39,312 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:12:39,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:12:41,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:12:42,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:47,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:50,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:12:52,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:12:54,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:12:54,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:12:54,312 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:12:58,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:00,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:04,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:09,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:09,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:13:09,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:13:09,774 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:13:10,021 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:13:12,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:14,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:19,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:22,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:24,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:13:24,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:13:24,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:13:24,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:29,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:30,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:13:30,527 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:13:30,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:13:30,529 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:13:31,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:13:32,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:33,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:35,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:39,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:13:39,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:13:39,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:13:39,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:13:41,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:41,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:46,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:51,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:13:52,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:13:54,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:13:54,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:13:54,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:13:58,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:01,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:14:03,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:08,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:09,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:14:09,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:14:09,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:14:10,022 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:14:11,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:14:14,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:19,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:21,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:14:24,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:14:24,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:24,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:14:24,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:14:29,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:31,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:14:35,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:39,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:14:39,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:14:39,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:14:39,937 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:14:40,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:43,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:14:45,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:50,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:14:52,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:14:54,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:14:54,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:14:54,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:14:55,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:00,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:02,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:15:06,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:09,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:15:10,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:15:10,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:15:10,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:15:11,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:12,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:15:16,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:21,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:23,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:15:25,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:15:25,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:15:25,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:15:27,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:33,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:15:33,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:38,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:39,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:15:41,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:15:41,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:15:41,363 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:15:43,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:15:43,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:48,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:53,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:15:53,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:15:56,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:15:56,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:15:56,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:15:58,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:03,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:04,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:09,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:09,932 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:16:11,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:16:11,360 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:16:11,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:16:13,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:20,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:23,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:25,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:26,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:16:26,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:16:26,366 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:16:31,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:33,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:36,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:39,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:16:41,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:16:41,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:16:41,361 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:16:41,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:43,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:47,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:51,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:16:51,718 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:16:51,719 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:16:51,722 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:16:52,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:16:52,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:16:53,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:54,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:16:56,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:16:56,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:16:56,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:16:58,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:03,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:03,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:17:08,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:09,946 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:17:11,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:17:11,360 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:17:11,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:17:13,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:13,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:17:18,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:22,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:17:23,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:26,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:17:26,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:17:26,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:17:29,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:31,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:17:35,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:39,935 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:17:41,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:17:41,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:41,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:17:41,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:17:41,366 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:17:46,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:51,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:17:51,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:17:56,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:17:56,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:17:56,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:17:57,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:01,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:18:03,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:08,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:09,935 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:18:11,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:18:11,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:18:11,360 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:18:11,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:18:14,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:19,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:21,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:18:24,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:27,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:18:27,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:18:27,269 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:18:29,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:18:30,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:35,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:39,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:18:39,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:18:40,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:42,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:18:42,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:18:42,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:18:46,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:49,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:18:51,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:56,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:18:57,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:18:57,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:18:57,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:18:57,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:19:02,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:07,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:19:08,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:09,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:19:12,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:19:12,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:19:12,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:19:13,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:17,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:19:18,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:23,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:27,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:19:27,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:19:27,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:19:29,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:31,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:19:35,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:39,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:19:41,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:41,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:19:42,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:19:42,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:19:42,267 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:19:46,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:51,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:52,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:19:56,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:19:58,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:19:58,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:19:58,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:20:01,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:02,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:07,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:08,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:20:08,834 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:20:08,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:20:08,837 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:20:09,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:20:09,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:09,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:20:10,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:12,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:13,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:20:13,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:20:13,268 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:20:18,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:19,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:23,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:28,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:20:28,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:20:28,268 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:20:29,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:29,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:34,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:39,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:39,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:39,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:20:43,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:20:43,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:20:43,270 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:20:45,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:49,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:20:50,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:55,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:20:58,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:20:58,269 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:20:58,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:20:59,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:01,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:06,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:09,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:09,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:21:11,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:13,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:21:13,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:21:13,269 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:21:17,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:17,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:22,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:27,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:28,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:21:28,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:28,269 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:21:28,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:21:33,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:37,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:39,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:39,954 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:21:43,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:21:43,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:21:43,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:21:44,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:47,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:49,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:54,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:21:58,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:21:58,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:21:58,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:21:58,270 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:22:00,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:05,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:08,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:22:09,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:22:10,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:13,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:22:13,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:22:13,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:22:16,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:18,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:22:21,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:26,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:28,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:22:28,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:22:28,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:22:28,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:22:32,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:37,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:22:37,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:39,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:22:42,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:43,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:22:43,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:22:43,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:22:47,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:22:48,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:53,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:22:57,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:22:58,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:22:58,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:22:58,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:22:59,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:04,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:09,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:09,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:23:13,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:23:13,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:23:13,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:23:15,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:20,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:23,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:23:26,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:28,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:23:28,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:23:28,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:23:31,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:33,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:23:36,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:39,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:23:41,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:43,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:23:43,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:23:43,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:23:43,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:23:43,474 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:23:43,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:23:43,479 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:23:43,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:23:44,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:23:45,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:23:47,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:52,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:56,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:23:57,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:23:58,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:23:58,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:23:58,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:24:03,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:06,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:24:09,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:09,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:24:13,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:24:13,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:24:13,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:24:14,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:16,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:24:19,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:24,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:26,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:24:28,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:24:28,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:24:28,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:24:30,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:36,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:24:36,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:39,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:24:41,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:43,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:24:43,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:24:43,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:24:46,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:24:47,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:52,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:55,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:24:58,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:24:58,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:24:58,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:24:58,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:25:03,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:05,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:25:08,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:09,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:25:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:14,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:25:14,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:25:14,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:25:17,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:25:19,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:24,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:27,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:25:29,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:25:29,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:25:29,682 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:25:29,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:34,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:38,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:25:39,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:25:39,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:44,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:45,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:25:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:25:45,361 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:25:48,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:25:50,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:55,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:25:58,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:26:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:26:00,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:26:01,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:07,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:08,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:09,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:26:12,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:15,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:26:15,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:26:15,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:26:18,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:18,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:23,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:28,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:29,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:30,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:26:30,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:26:30,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:26:34,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:36,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:39,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:26:39,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:44,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:45,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:26:45,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:26:45,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:26:46,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:50,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:26:55,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:26:55,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:00,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:27:00,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:27:00,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:27:01,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:05,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:27:05,352 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:27:05,352 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:27:05,354 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:27:05,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:27:06,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:27:07,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:07,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:27:09,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:27:12,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:15,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:27:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:27:15,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:27:17,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:27:18,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:23,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:28,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:27:29,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:30,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:27:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:27:30,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:27:35,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:38,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:27:39,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:27:40,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:45,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:27:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:27:45,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:27:46,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:48,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:27:51,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:57,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:27:58,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:28:00,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:28:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:28:00,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:28:02,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:07,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:08,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:28:09,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:28:12,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:15,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:28:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:28:15,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:28:18,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:28:18,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:23,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:26,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:28:29,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:30,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:28:30,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:28:30,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:28:34,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:38,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:28:39,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:39,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:28:44,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:28:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:28:45,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:28:48,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:28:50,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:55,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:28:58,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:29:00,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:29:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:29:00,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:29:00,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:05,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:08,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:29:09,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:29:10,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:15,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:29:15,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:29:15,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:29:16,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:18,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:29:21,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:27,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:28,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:29:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:29:30,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:29:30,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:29:32,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:38,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:38,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:29:39,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:29:43,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:29:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:29:45,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:29:46,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:29:49,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:55,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:29:56,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:30:00,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:30:00,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:30:01,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:06,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:07,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:09,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:30:12,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:15,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:30:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:30:15,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:30:16,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:18,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:23,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:23,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:30:23,816 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:30:23,816 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:30:23,818 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:30:23,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:30:25,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:28,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:30:30,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:30:30,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:30:34,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:35,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:39,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:39,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:30:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:30:45,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:30:45,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:30:45,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:46,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:50,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:30:56,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:30:56,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:31:00,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:31:00,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:31:02,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:06,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:31:08,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:09,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:31:13,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:31:15,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:31:15,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:31:16,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:31:19,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:24,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:31:24,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:29,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:30,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:31:30,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:31:30,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:31:34,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:31:35,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:39,967 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:31:40,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:44,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:31:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:31:45,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:31:45,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:31:46,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:51,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:31:54,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:31:57,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:32:00,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:32:00,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:32:02,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:07,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:09,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:32:12,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:14,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:32:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:32:15,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:32:15,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:32:18,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:24,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:24,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:32:29,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:31,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:32:31,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:32:31,513 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:32:34,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:35,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:32:39,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:39,969 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:32:45,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:32:45,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:46,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:32:46,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:32:46,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:32:50,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:32:54,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:32:56,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:01,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:33:01,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:33:01,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:33:02,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:02,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:07,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:09,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:33:12,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:13,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:16,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:33:16,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:33:16,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:33:18,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:22,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:24,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:29,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:31,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:33:31,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:33:31,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:33:32,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:34,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:39,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:39,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:33:42,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:45,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:46,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:33:46,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:33:46,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:33:49,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:33:49,977 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:33:49,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:33:49,979 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:33:50,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:33:50,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:33:51,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:52,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:33:55,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:00,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:34:01,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:01,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:34:01,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:34:01,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:34:06,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:09,984 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:34:10,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:34:11,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:16,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:17,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:34:17,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:34:17,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:34:20,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:34:22,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:27,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:30,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:34:32,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:34:32,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:34:32,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:34:32,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:37,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:39,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:34:43,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:44,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:34:47,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:34:47,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:34:47,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:34:48,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:54,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:34:54,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:34:59,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:02,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:35:02,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:35:02,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:35:04,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:05,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:35:09,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:09,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:35:13,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:35:15,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:17,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:35:17,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:35:17,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:35:20,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:23,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:35:26,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:31,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:32,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:35:32,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:35:32,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:35:33,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:35:36,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:39,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:35:42,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:42,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:35:47,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:47,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:35:47,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:35:47,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:35:52,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:35:52,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:35:57,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:02,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:36:02,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:36:02,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:36:02,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:36:02,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:07,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:09,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:36:12,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:36:12,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:17,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:36:17,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:36:17,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:36:18,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:22,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:36:24,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:29,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:32,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:36:32,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:36:32,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:36:32,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:36:34,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:39,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:39,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:36:42,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:36:45,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:47,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:36:47,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:36:47,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:36:51,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:36:52,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:36:56,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:01,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:02,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:37:02,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:37:02,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:37:02,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:06,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:09,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:37:10,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:37:10,057 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:37:10,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:37:10,061 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:37:11,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:37:11,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:12,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:12,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:17,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:18,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:37:18,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:37:18,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:37:21,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:22,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:27,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:31,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:33,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:37:33,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:37:33,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:37:38,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:39,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:37:41,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:43,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:48,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:37:48,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:37:48,712 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:37:49,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:37:51,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:37:54,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:00,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:01,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:38:03,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:38:03,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:38:03,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:38:05,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:09,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:38:11,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:11,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:38:16,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:18,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:38:18,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:38:18,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:38:21,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:38:21,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:26,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:30,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:38:32,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:33,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:38:33,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:38:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:38:37,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:39,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:38:40,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:38:42,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:47,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:38:48,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:38:48,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:38:50,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:38:53,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:38:58,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:00,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:39:03,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:03,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:39:03,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:39:03,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:39:08,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:09,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:39:10,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:39:13,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:18,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:39:18,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:39:18,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:39:19,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:20,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:39:24,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:30,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:30,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:39:33,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:39:33,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:39:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:39:35,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:39,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:39:40,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:41,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:39:45,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:48,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:39:48,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:39:48,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:39:51,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:39:51,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:39:56,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:01,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:02,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:03,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:40:03,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:40:03,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:40:07,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:09,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:40:11,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:12,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:17,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:18,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:40:18,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:40:18,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:40:21,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:23,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:28,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:40:28,247 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:40:28,247 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:40:28,250 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:40:28,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:40:29,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:29,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:30,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:33,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:40:33,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:40:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:40:34,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:39,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:39,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:40:39,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:44,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:48,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:40:48,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:40:48,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:40:49,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:40:50,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:55,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:40:58,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:41:01,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:03,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:41:03,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:41:03,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:41:06,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:08,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:41:09,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:41:11,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:17,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:18,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:41:18,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:41:18,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:41:18,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:41:22,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:27,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:28,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:41:32,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:33,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:41:33,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:41:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:41:38,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:39,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:41:39,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:41:43,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:41:48,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:41:48,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:41:49,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:41:49,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:54,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:41:59,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:42:00,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:42:03,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:42:03,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:42:05,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:09,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:42:09,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:42:10,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:16,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:18,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:42:18,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:42:18,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:42:19,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:42:22,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:27,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:29,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:42:32,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:33,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:42:33,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:42:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:42:37,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:39,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:42:40,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:42:43,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:48,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:42:48,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:42:48,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:42:49,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:42:53,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:59,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:42:59,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:43:03,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:43:03,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:43:04,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:09,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:09,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:43:10,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:15,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:18,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:43:18,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:43:18,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:43:19,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:20,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:25,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:28,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:31,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:33,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:43:33,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:43:33,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:43:36,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:40,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:43:40,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:42,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:47,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:48,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:43:48,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:43:48,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:43:49,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:43:49,016 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:43:49,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:43:49,019 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:43:49,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:43:50,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:51,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:43:52,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:43:57,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:01,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:44:02,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:03,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:44:03,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:44:03,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:44:07,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:10,001 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:44:11,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:44:13,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:18,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:18,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:44:18,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:44:18,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:44:21,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:44:23,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:28,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:31,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:44:33,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:44:33,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:44:33,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:44:34,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:39,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:40,001 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:44:41,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:44:45,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:49,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:44:49,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:44:49,315 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:44:51,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:44:51,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:44:56,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:01,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:03,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:45:04,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:45:04,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:45:04,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:45:07,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:10,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:45:13,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:13,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:45:18,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:20,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:45:20,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:45:20,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:45:23,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:23,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:45:28,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:33,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:45:34,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:35,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:45:35,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:45:35,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:45:39,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:40,004 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:45:43,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:45:45,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:50,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:45:50,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:45:50,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:45:50,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:45:54,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:45:56,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:01,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:03,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:46:05,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:46:05,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:46:05,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:46:07,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:10,005 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:46:12,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:13,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:46:17,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:46:20,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:46:20,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:46:23,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:46:23,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:28,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:33,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:46:34,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:35,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:46:35,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:46:35,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:46:40,006 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:46:40,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:41,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:46:45,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:50,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:46:50,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:46:50,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:46:51,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:46:53,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:46:56,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:01,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:01,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:05,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:47:05,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:47:05,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:47:07,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:10,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:47:10,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:47:10,486 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:47:10,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:47:10,490 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:47:10,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:47:11,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:12,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:12,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:17,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:20,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:47:20,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:47:20,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:47:21,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:22,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:28,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:31,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:33,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:35,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:47:35,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:47:35,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:47:38,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:40,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:47:42,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:44,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:49,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:50,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:47:50,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:47:50,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:47:52,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:47:54,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:47:59,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:02,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:48:04,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:05,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:48:05,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:48:05,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:48:09,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:10,008 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:48:11,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:48:15,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:20,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:20,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:48:20,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:48:20,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:48:21,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:48:25,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:30,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:31,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:48:35,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:35,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:48:35,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:48:35,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:48:40,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:48:41,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:48:41,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:46,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:50,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:48:50,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:48:50,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:48:51,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:48:52,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:48:57,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:01,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:49:02,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:05,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:49:05,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:49:05,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:49:08,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:10,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:49:11,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:49:13,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:18,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:49:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:49:20,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:49:24,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:25,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:49:29,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:35,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:35,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:49:35,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:49:35,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:49:35,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:49:40,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:49:41,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:46,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:47,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:49:50,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:49:50,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:49:50,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:49:51,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:56,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:49:58,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:50:01,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:05,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:50:05,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:50:05,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:50:06,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:08,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:50:10,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:50:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:17,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:18,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:50:20,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:50:20,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:50:20,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:50:23,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:28,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:50:28,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:33,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:35,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:50:35,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:50:35,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:50:37,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:50:37,186 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:50:37,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:50:37,197 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:50:37,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:50:38,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:50:39,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:40,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:50:45,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:48,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:50:50,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:50:50,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:50:50,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:50,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:50:56,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:50:57,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:51:02,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:05,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:51:05,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:51:05,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:51:07,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:51:07,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:10,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:51:13,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:17,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:51:18,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:20,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:51:20,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:51:20,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:51:24,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:27,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:51:29,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:34,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:35,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:51:35,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:51:35,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:51:37,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:51:39,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:40,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:51:45,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:48,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:51:50,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:50,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:51:50,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:51:50,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:51:55,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:51:58,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:52:00,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:05,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:52:05,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:52:05,909 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:52:06,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:08,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:52:10,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:52:12,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:17,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:18,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:52:20,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:52:20,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:52:20,909 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:52:22,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:27,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:28,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:52:32,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:35,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:52:35,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:52:35,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:52:36,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:52:38,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:40,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:52:44,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:46,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:52:49,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:50,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:52:50,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:52:50,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:52:55,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:52:56,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:00,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:05,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:05,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:53:05,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:53:05,909 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:53:06,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:10,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:53:12,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:15,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:17,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:20,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:53:20,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:53:20,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:53:22,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:25,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:28,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:33,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:35,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:53:35,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:53:35,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:53:35,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:38,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:40,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:53:44,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:45,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:49,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:50,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:53:50,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:53:50,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:53:54,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:53:54,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:53:54,495 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:53:54,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:53:54,499 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:53:55,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:53:56,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:57,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:53:59,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:05,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:05,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:54:05,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:54:05,909 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:54:06,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:54:10,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:54:11,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:16,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:54:16,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:21,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:22,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:54:22,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:54:22,265 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:54:24,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:54:27,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:32,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:34,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:54:36,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:54:37,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:54:37,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:54:37,263 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:54:38,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:40,015 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:54:44,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:46,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:54:49,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:53,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:54:53,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:54:53,221 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:54:55,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:54:56,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:55:00,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:06,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:06,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:55:08,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:55:08,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:55:08,218 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:55:10,015 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:55:12,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:16,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:55:17,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:22,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:23,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:55:23,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:55:23,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:55:26,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:55:27,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:32,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:35,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:55:38,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:55:38,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:55:38,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:55:38,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:40,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:55:44,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:45,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:55:49,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:53,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:55:53,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:55:53,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:55:55,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:55:55,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:56:00,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:05,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:56:06,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:08,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:56:08,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:56:08,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:56:10,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:56:12,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:16,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:56:17,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:22,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:23,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:56:23,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:56:23,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:56:26,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:56:28,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:33,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:36,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:56:38,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:56:38,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:56:38,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:56:38,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:40,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:56:44,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:46,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:56:49,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:53,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:56:53,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:56:53,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:56:55,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:56:56,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:00,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:05,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:06,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:08,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:57:08,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:57:08,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:57:10,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:57:11,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:13,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 14:57:13,457 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 14:57:13,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 14:57:13,459 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 14:57:13,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 14:57:14,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:15,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:16,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:21,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:23,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:57:23,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:57:23,683 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:57:24,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:26,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:31,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:36,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:37,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:38,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:57:38,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:57:38,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:57:40,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:57:43,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:46,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:48,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:53,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:57:53,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:57:53,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:57:53,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:57:54,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:57:58,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:04,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:04,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:58:08,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:58:08,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:58:08,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:58:09,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:10,020 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:58:14,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:58:15,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:20,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:23,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:58:23,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:58:23,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:58:23,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:58:25,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:30,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:34,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:58:36,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:38,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:58:38,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:58:38,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:58:40,020 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:58:42,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:44,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:58:47,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:53,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:58:53,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:58:53,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:58:53,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:58:54,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:58:58,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:04,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:04,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:59:08,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:59:08,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:59:08,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:59:09,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:10,021 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:59:14,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:59:15,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:20,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:23,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:59:23,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:59:23,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:59:24,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:59:25,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:30,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:34,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:59:36,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:38,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:59:38,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:59:38,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:59:40,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 14:59:42,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:44,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:59:47,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:52,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 14:59:53,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 14:59:53,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 14:59:53,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 14:59:53,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 14:59:58,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:02,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:00:04,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:09,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:09,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:00:09,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:00:09,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:00:10,103 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:00:12,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:00:15,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:20,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:22,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:00:24,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:00:24,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:00:24,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:00:25,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:30,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:31,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:00:31,087 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:00:31,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:00:31,089 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:00:31,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:00:32,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:00:36,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:39,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:00:39,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:00:39,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:00:40,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:00:41,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:42,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:00:46,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:52,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:00:53,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:00:54,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:00:54,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:00:54,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:00:58,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:03,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:01:03,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:08,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:09,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:01:09,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:01:09,933 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:01:10,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:01:13,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:01:14,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:19,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:22,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:01:24,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:24,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:01:24,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:01:24,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:01:30,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:32,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:01:35,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:39,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:01:39,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:01:39,932 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:01:40,129 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:01:41,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:42,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:01:46,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:52,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:01:52,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:01:54,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:01:54,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:01:54,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:01:57,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:02,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:02:02,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:07,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:09,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:02:09,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:02:09,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:02:10,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:02:10,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:02:13,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:18,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:20,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:02:23,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:24,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:02:24,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:02:24,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:02:29,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:30,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:02:34,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:39,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:39,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:02:39,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:02:39,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:02:40,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:02:40,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:02:45,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:50,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:02:50,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:02:54,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:02:54,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:02:54,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:02:56,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:01,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:03:01,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:06,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:09,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:03:09,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:03:09,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:03:10,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:03:11,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:03:12,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:17,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:21,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:03:23,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:24,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:03:24,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:03:24,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:03:28,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:31,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:03:33,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:38,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:39,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:03:39,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:03:39,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:03:39,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:03:40,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:03:44,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:49,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:53,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:03:53,016 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:03:53,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:03:53,021 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:03:53,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:03:53,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:03:54,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:03:54,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:03:54,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:03:55,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:03:55,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:00,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:05,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:06,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:09,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:04:09,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:04:09,935 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:04:10,178 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:04:11,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:14,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:16,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:21,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:24,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:24,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:04:24,932 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:04:24,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:04:27,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:32,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:34,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:39,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:04:39,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:04:39,937 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:04:40,125 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:04:43,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:44,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:48,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:54,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:04:54,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:04:54,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:04:54,933 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:04:54,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:05:00,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:02,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:05:06,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:10,033 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:05:10,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:05:10,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:05:10,906 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:05:11,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:14,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:05:16,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:21,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:23,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:05:25,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:05:25,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:05:25,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:05:26,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:31,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:33,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:05:36,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:40,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:05:40,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:05:40,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:05:40,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:05:41,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:43,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:05:46,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:52,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:05:53,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:05:55,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:05:55,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:05:55,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:05:58,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:03,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:03,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:06:08,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:10,034 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:06:10,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:06:10,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:06:10,905 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:06:13,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:06:14,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:19,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:23,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:06:24,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:25,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:06:25,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:06:25,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:06:30,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:33,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:06:35,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:40,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:06:40,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:06:40,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:06:40,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:06:41,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:43,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:06:46,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:50,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:06:51,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:06:55,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:06:55,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:06:55,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:06:57,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:00,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:03,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:08,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:09,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:07:09,644 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:07:09,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:07:09,648 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:07:09,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:07:10,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:07:10,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:10,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:07:10,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:07:10,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:07:11,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:13,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:18,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:20,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:23,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:25,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:07:25,905 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:07:25,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:07:28,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:30,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:33,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:38,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:40,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:07:40,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:07:40,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:07:40,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:07:41,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:44,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:49,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:51,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:07:54,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:07:55,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:07:55,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:07:55,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:08:00,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:01,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:08:05,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:10,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:08:10,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:10,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:08:10,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:08:10,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:08:13,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:08:15,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:20,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:23,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:08:25,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:25,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:08:25,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:08:25,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:08:31,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:08:31,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:36,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:40,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:08:40,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:08:40,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:08:40,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:08:41,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:08:42,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:47,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:51,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:08:52,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:08:55,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:08:55,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:08:55,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:08:58,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:01,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:09:03,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:08,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:10,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:09:10,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:09:10,905 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:09:10,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:09:11,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:09:14,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:19,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:21,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:09:24,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:27,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:09:27,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:09:27,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:09:29,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:30,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:09:34,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:40,038 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:09:40,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:40,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:09:42,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:09:42,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:09:42,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:09:45,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:50,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:51,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:09:55,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:09:57,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:09:57,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:09:57,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:09:59,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:01,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:06,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:09,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:10,039 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:10:12,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:12,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:10:12,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:10:12,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:10:17,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:19,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:23,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:27,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:10:27,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:10:27,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:10:28,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:28,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:10:28,489 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:10:28,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:10:28,494 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:10:29,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:10:29,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:30,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:33,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:39,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:39,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:40,039 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:10:42,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:10:42,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:10:42,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:10:44,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:49,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:10:50,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:55,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:10:57,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:10:57,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:10:57,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:10:59,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:01,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:06,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:09,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:10,040 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:11:12,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:12,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:11:12,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:11:12,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:11:17,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:19,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:22,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:27,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:11:27,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:11:27,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:11:27,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:29,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:32,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:37,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:38,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:40,040 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:11:42,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:11:42,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:11:42,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:11:43,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:47,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:49,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:54,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:11:57,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:11:57,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:11:57,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:11:57,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:11:59,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:04,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:07,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:12:09,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:10,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:12:12,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:12:12,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:12:12,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:12:15,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:18,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:12:20,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:26,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:27,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:12:27,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:12:27,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:12:27,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:12:32,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:37,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:12:38,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:40,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:12:42,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:12:42,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:12:42,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:12:43,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:49,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:49,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:12:54,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:12:57,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:12:57,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:12:57,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:12:57,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:12:59,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:04,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:07,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:10,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:13:10,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:12,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:13:12,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:13:12,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:13:15,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:17,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:21,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:26,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:27,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:13:27,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:13:27,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:13:27,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:32,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:37,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:37,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:40,043 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:13:42,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:13:42,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:13:42,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:13:43,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:45,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:13:45,766 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:13:45,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:13:45,769 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:13:46,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:13:47,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:48,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:48,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:53,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:13:57,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:13:57,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:13:57,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:13:57,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:13:59,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:04,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:07,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:14:10,043 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:14:10,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:14,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:14:14,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:14:14,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:14:15,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:17,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:14:20,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:25,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:27,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:14:29,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:14:29,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:14:29,013 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:14:31,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:37,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:38,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:14:40,044 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:14:43,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:44,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:14:44,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:14:44,696 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:14:48,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:14:48,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:53,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:58,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:14:58,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:14:59,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:14:59,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:14:59,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:15:04,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:08,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:15:10,044 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:15:10,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:15,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:15:15,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:15:15,223 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:15:17,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:15:20,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:25,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:27,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:15:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:15:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:15:30,176 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:15:31,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:36,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:39,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:15:40,045 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:15:42,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:46,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:15:46,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:15:46,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:15:47,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:49,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:15:52,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:57,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:15:59,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:16:01,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:16:01,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:16:01,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:16:03,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:09,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:09,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:16:10,045 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:16:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:17,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:16:17,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:16:17,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:16:19,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:16:20,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:25,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:29,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:16:30,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:32,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:16:32,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:16:32,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:16:36,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:39,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:16:40,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:16:42,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:47,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:16:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:16:47,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:16:50,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:16:52,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:16:57,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:00,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:02,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:17:02,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:17:02,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:17:03,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:07,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:17:07,606 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:17:07,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:17:07,611 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:17:08,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:17:08,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:08,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:10,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:17:10,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:14,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:17,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:17:17,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:17:17,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:17:18,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:19,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:24,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:28,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:29,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:32,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:17:32,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:17:32,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:17:35,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:38,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:40,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:17:41,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:46,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:47,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:17:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:17:47,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:17:48,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:17:51,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:17:57,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:18:02,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:18:02,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:18:02,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:18:02,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:07,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:18:08,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:10,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:18:14,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:17,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:18:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:18:17,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:18:17,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:18:19,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:24,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:27,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:18:30,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:32,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:18:32,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:18:32,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:18:35,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:40,048 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:18:40,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:41,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:18:45,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:18:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:18:47,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:18:51,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:18:51,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:18:56,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:01,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:19:02,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:19:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:19:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:02,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:19:07,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:10,049 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:19:12,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:19:12,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:17,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:19:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:19:17,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:19:18,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:22,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:19:23,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:28,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:32,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:19:32,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:19:32,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:19:32,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:19:34,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:39,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:40,050 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:19:44,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:19:44,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:19:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:19:47,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:19:50,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:19:54,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:19:55,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:00,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:02,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:20:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:20:02,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:20:02,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:20:06,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:10,052 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:20:11,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:14,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:20:16,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:20:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:20:17,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:20:22,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:22,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:20:27,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:31,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:20:31,117 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:20:31,117 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:20:31,120 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:20:31,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:20:32,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:20:32,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:20:32,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:20:32,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:32,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:20:37,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:40,053 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:20:42,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:20:43,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:20:47,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:20:47,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:20:49,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:51,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:20:54,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:20:59,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:01,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:21:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:21:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:21:02,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:21:05,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:10,053 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:21:10,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:11,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:21:15,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:21:17,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:21:17,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:21:20,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:21,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:21:25,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:30,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:32,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:21:32,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:21:32,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:21:32,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:21:36,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:40,055 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:21:42,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:42,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:21:47,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:21:47,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:21:47,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:21:52,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:21:54,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:21:57,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:22:02,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:22:02,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:22:03,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:04,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:22:08,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:10,056 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:22:12,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:22:13,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:17,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:22:17,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:22:17,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:22:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:24,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:22:24,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:29,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:32,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:22:32,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:22:32,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:22:32,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:22:35,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:40,057 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:22:41,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:42,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:22:46,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:47,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:22:47,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:22:47,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:22:52,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:22:52,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:22:57,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:23:02,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:23:02,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:23:02,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:02,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:07,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:10,058 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:23:12,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:12,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:17,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:23:17,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:23:17,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:23:18,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:22,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:23,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:28,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:32,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:23:32,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:23:32,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:23:32,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:34,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:39,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:40,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:23:41,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:45,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:47,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:23:47,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:23:47,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:23:50,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:23:51,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:23:51,666 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:23:51,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:23:51,668 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:23:52,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:23:52,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:53,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:23:55,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:00,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:02,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:24:02,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:24:02,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:24:02,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:24:06,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:10,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:24:11,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:12,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:24:16,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:17,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:24:17,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:24:17,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:24:22,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:24:22,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:27,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:32,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:24:32,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:24:32,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:24:32,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:24:33,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:38,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:40,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:24:42,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:24:43,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:47,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:24:47,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:24:47,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:24:49,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:24:52,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:24:55,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:00,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:02,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:25:02,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:25:02,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:25:02,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:25:05,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:10,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:25:11,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:14,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:25:16,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:17,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:25:17,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:25:17,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:25:21,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:24,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:25:26,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:31,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:32,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:25:32,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:25:32,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:25:34,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:25:37,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:40,063 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:25:43,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:44,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:25:47,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:25:47,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:25:47,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:25:49,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:25:54,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:25:55,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:00,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:02,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:26:02,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:26:02,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:26:03,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:26:05,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:10,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:26:11,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:15,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:26:16,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:17,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:26:17,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:26:17,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:26:21,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:23,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:26:26,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:32,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:26:32,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:26:32,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:32,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:26:32,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:26:37,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:40,067 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:26:42,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:26:42,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:47,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:26:47,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:26:47,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:26:48,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:52,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:26:54,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:26:59,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:02,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:27:02,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:27:02,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:27:02,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:27:04,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:09,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:10,067 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:27:10,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:27:10,847 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:27:10,847 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:27:10,850 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:27:11,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:27:12,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:27:13,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:27:14,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:17,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:27:17,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:27:17,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:27:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:22,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:27:25,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:30,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:32,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:27:32,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:27:32,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:27:32,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:27:36,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:40,069 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:27:41,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:42,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:27:46,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:47,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:27:47,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:27:47,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:27:51,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:27:56,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:00,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:28:02,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:02,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:28:02,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:28:02,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:28:07,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:10,069 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:28:10,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:28:13,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:18,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:18,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:28:18,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:28:18,550 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:28:21,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:28:23,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:28,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:31,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:28:33,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:28:33,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:28:33,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:28:34,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:40,070 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:28:40,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:41,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:28:45,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:48,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:28:48,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:28:48,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:28:50,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:28:51,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:28:55,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:01,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:01,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:03,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:29:03,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:29:03,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:29:06,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:10,070 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:29:11,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:12,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:17,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:18,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:29:18,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:29:18,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:29:20,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:22,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:27,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:28,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:33,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:33,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:29:33,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:29:33,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:29:38,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:39,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:40,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:29:44,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:48,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:48,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:29:48,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:29:48,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:29:49,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:54,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:29:58,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:29:59,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:03,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:30:03,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:30:03,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:30:05,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:08,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:30:10,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:30:11,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:16,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:18,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:30:18,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:30:18,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:30:18,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:30:21,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:27,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:28,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:30:32,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:33,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:30:33,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:30:33,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:30:37,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:30:37,206 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:30:37,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:30:37,208 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:30:37,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:30:38,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:38,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:30:39,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:30:40,075 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:30:44,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:48,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:30:48,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:30:48,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:30:49,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:30:49,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:54,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:30:59,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:31:00,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:03,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:31:03,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:31:03,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:31:05,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:09,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:31:10,072 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:31:11,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:16,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:18,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:31:18,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:31:18,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:31:19,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:31:21,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:26,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:29,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:31:32,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:33,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:31:33,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:31:33,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:31:37,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:39,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:31:40,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:31:43,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:48,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:48,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:31:48,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:31:48,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:31:49,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:31:53,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:59,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:31:59,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:32:03,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:32:03,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:32:03,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:32:04,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:09,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:32:10,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:32:10,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:15,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:18,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:32:18,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:32:18,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:32:19,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:32:20,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:25,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:29,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:32:31,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:33,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:32:33,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:32:33,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:32:36,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:38,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:32:40,085 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:32:42,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:47,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:48,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:32:48,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:32:48,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:32:48,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:32:52,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:57,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:32:58,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:33:03,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:03,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:33:03,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:33:03,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:33:08,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:08,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:33:10,074 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:33:14,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:18,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:33:18,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:33:18,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:33:19,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:33:19,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:24,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:29,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:33:29,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:33,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:33:33,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:33:33,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:33:35,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:40,076 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:33:41,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:43,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:33:46,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:48,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:33:48,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:33:48,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:33:51,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:33:53,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:33:57,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:02,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:02,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:34:02,290 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:34:02,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:34:02,292 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:34:02,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:34:03,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:34:03,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:34:03,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:34:03,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:34:05,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:34:07,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:10,076 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:34:13,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:18,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:18,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:34:18,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:34:18,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:34:23,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:28,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:33,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:34:33,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:34:33,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:34:33,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:38,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:40,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:34:44,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:34:48,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:34:48,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:34:49,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:54,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:34:59,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:03,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:35:03,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:35:03,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:35:04,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:09,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:10,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:35:15,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:18,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:35:18,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:35:18,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:35:20,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:25,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:30,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:33,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:35:33,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:35:33,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:35:36,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:40,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:35:42,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:47,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:35:48,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:35:48,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:35:52,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:35:57,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:02,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:03,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:36:03,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:36:03,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:36:07,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:10,078 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:36:13,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:18,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:18,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:36:18,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:36:18,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:36:23,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:28,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:33,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:36:33,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:36:33,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:36:34,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:39,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:40,078 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:36:45,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:36:48,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:36:48,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:36:50,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:36:55,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:00,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:03,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:37:03,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:37:03,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:37:05,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:10,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:37:11,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:16,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:18,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:37:18,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:37:18,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:37:21,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:26,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:31,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:33,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:37:33,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:37:33,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:37:37,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:40,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:37:43,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:48,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:48,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:37:48,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:37:48,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:37:53,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:37:58,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:03,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:38:03,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:38:03,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:38:04,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:09,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:10,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:38:15,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:18,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:38:18,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:38:18,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:38:20,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:25,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:30,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:33,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:38:33,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:38:33,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:38:35,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:40,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:38:41,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:46,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:48,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:38:48,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:38:48,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:38:51,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:38:56,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:01,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:03,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:39:03,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:39:03,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:39:06,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:10,081 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:39:12,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:17,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:18,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:39:18,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:39:18,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:39:22,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:27,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:32,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:33,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:39:33,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:39:33,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:39:38,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:40,081 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:39:44,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:48,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:39:48,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:39:48,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:39:49,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:54,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:39:59,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:03,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:40:03,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:40:03,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:40:05,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:10,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:40:11,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:16,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:18,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:40:18,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:40:18,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:40:21,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:26,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:31,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:33,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:40:33,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:40:33,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:40:37,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:40,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:40:43,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:48,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:48,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:40:48,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:40:48,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:40:53,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:40:58,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:03,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:41:03,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:41:03,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:41:04,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:09,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:10,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:41:15,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:18,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:41:18,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:41:18,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:41:20,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:30,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:33,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:41:33,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:41:33,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:41:35,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:40,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:41:41,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:46,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:48,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:41:48,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:41:48,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:41:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:41:56,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:01,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:03,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:42:03,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:42:03,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:42:07,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:10,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:42:13,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:18,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:18,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:42:18,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:42:18,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:42:23,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:28,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:33,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:42:33,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:42:33,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:42:34,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:39,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:40,085 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:42:45,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:48,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:42:48,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:42:48,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:42:50,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:42:55,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:00,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:03,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:43:03,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:43:03,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:43:05,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:10,085 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:43:11,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:16,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:18,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:43:18,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:43:18,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:43:21,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:26,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:31,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:33,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:43:33,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:43:33,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:43:36,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:40,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:43:42,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:47,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:48,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:43:48,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:43:48,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:43:52,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:43:57,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:02,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:03,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:44:03,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:44:03,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:44:08,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:10,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:44:14,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:14,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:44:14,931 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 15:44:14,933 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 15:44:14,933 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 15:44:14,933 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 15:44:14,934 DEBUG SenderThread:607337 [sender.py:send():380] send: metric -2023-11-19 15:44:14,934 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:44:14,935 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:44:14,938 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:44:15,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:44:16,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:44:18,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:44:19,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:44:19,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:44:19,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:44:19,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:24,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:29,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:34,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:35,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:44:35,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:44:35,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:44:38,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:44:40,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:44:40,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:41,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/config.yaml -2023-11-19 15:44:42,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:44:45,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:50,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:44:55,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:05,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:19,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:19,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 15:45:19,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:45:19,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:45:19,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:45:21,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:45:25,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:27,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 15:45:30,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:35,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:40,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:45,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:45,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:45:45,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:45:45,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:45:48,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:45:49,872 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:45:50,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:45:51,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:45:51,333 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:45:51,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:45:56,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:01,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:06,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:46:06,332 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:46:06,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:46:06,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:11,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:16,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:19,872 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:46:21,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:46:21,332 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:46:21,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:46:22,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:27,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:32,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:36,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:46:36,332 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:46:36,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:46:38,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:48,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:46:48,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:46:49,873 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:46:51,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:46:51,332 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:46:51,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:46:54,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:00,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:47:00,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:05,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:06,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:47:06,333 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:47:06,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:47:10,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:12,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:47:15,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:19,875 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:47:20,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:21,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:47:21,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:47:21,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:47:24,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:47:26,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:34,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:47:36,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:47:36,333 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:47:36,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:47:36,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:41,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:44,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:47:47,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:49,877 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:47:51,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:47:51,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:47:51,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:47:53,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:47:54,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:47:58,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:03,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:04,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:48:06,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:48:06,334 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:48:06,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:48:09,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:14,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:48:14,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:19,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:48:20,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:21,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:48:21,336 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:48:21,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:48:23,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:48:26,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:31,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:34,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:48:36,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:48:36,334 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:48:36,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:48:36,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:41,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:46,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:48:46,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:49,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:48:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:48:52,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:48:52,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:48:52,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:48:56,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:48:57,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:02,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:04,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:49:07,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:49:07,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:49:07,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:49:07,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:12,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:16,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:49:18,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:19,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:49:22,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:49:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:49:22,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:49:24,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:24,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:49:29,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:34,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:49:34,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:37,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:49:37,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:49:37,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:49:40,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:44,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:49:46,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:49,883 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:49:51,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:49:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:49:52,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:49:52,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:49:54,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:49:57,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:02,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:04,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:50:07,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:07,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:50:07,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:50:07,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:50:12,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:12,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:50:12,828 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:50:12,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:50:12,832 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:50:13,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:50:14,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:50:17,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:19,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:50:22,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:50:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:50:22,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:50:22,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:24,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:50:27,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:33,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:34,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:50:37,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:50:37,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:50:37,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:50:38,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:44,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:45,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:50:49,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:49,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:50:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:50:52,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:50:52,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:50:54,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:50:55,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:50:59,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:04,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:51:05,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:07,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:51:07,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:51:07,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:51:10,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:12,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:51:16,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:19,886 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:51:22,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:22,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:51:22,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:51:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:51:22,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:51:27,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:32,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:51:33,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:37,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:51:37,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:51:37,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:51:38,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:44,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:51:44,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:49,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:49,888 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:51:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:51:52,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:51:52,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:51:54,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:51:54,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:51:59,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:04,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:52:05,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:07,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:52:07,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:52:07,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:52:10,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:14,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:52:16,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:19,889 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:52:21,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:52:22,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:52:22,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:52:24,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:52:26,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:31,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:35,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:52:37,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:37,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:52:37,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:52:37,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:52:43,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:45,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:52:48,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:49,892 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:52:52,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:52:52,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:52:52,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:52:53,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:52:55,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:52:58,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:03,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:05,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:53:07,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:53:07,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:53:07,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:53:09,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:12,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:53:15,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:19,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:53:20,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:22,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:53:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:53:22,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:53:22,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:53:26,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:31,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:53:31,537 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:53:31,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:53:31,541 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:53:32,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:53:32,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:53:32,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:37,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:37,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:53:37,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:53:37,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:53:42,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:44,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:53:47,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:49,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:53:52,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:53:52,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:53:52,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:53:52,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:53:54,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:53:57,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:03,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:04,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:54:07,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:54:07,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:54:07,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:54:08,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:14,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:14,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:54:19,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:19,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:54:22,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:54:22,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:54:22,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:54:24,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:24,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:54:29,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:34,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:54:35,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:37,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:54:37,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:54:37,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:54:40,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:45,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:54:46,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:49,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:54:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:54:52,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:54:52,727 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:54:52,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:54:57,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:54:57,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:02,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:07,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:55:07,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:07,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:55:07,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:55:07,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:55:12,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:17,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:55:18,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:19,895 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:55:23,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:55:23,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:55:23,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:55:23,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:27,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:55:29,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:34,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:37,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:55:38,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:55:38,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:55:38,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:55:40,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:46,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:46,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:55:49,895 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:55:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:55:54,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:55:54,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:55:54,223 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:55:56,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:55:57,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:02,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:07,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:08,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:56:09,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:56:09,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:56:09,219 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:56:12,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:17,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:18,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:56:19,896 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:56:22,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:24,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:56:24,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:56:24,220 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:56:28,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:33,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:38,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:39,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:56:39,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:56:39,220 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:56:43,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:48,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:49,897 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:56:53,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:56:54,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:56:54,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:56:54,220 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:56:55,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:56:59,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:04,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:05,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:57:09,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:57:09,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:57:09,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:57:10,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:15,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:57:16,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:19,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:57:21,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:21,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 15:57:21,981 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 15:57:21,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 15:57:21,985 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 15:57:22,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 15:57:23,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:57:24,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:57:24,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:57:24,220 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:57:27,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:33,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:33,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:57:38,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:39,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:57:39,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:57:39,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:57:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:45,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:57:48,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:49,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:57:54,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:57:54,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:57:54,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:57:54,221 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:57:55,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:57:59,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:03,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:58:05,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:10,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:10,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:58:10,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:58:10,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:58:14,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:58:15,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:19,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:58:20,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:22,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:58:25,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:58:25,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:58:25,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:58:26,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:31,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:32,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:58:36,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:40,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:58:40,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:58:40,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:58:42,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:42,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:58:47,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:49,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:58:53,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:58:53,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:58:55,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:58:55,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:58:55,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:58:59,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:03,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:59:04,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:09,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:10,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:59:10,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:59:10,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:59:11,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:59:15,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:19,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:59:20,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:21,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:59:25,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:59:25,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:59:25,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:59:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:31,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:59:32,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:37,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:40,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:59:40,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:59:40,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:59:41,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:59:42,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:47,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:49,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 15:59:51,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 15:59:52,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 15:59:57,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 15:59:57,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 15:59:58,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 15:59:59,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:03,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:08,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:09,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:00:12,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:00:12,013 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:00:14,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:19,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:19,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:19,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:00:24,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:27,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:00:27,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:00:27,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:00:29,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:30,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:35,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:00:36,546 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:00:36,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:00:36,548 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:00:36,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:00:39,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:40,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:42,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:00:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:00:42,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:00:46,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:48,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:49,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:00:51,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:56,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:00:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:00:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:00:57,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:00:57,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:01:02,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:07,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:01:12,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:01:12,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:01:13,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:01:13,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:18,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:19,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:01:23,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:01:23,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:27,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:01:27,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:01:27,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:01:29,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:35,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:35,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:01:40,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:01:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:01:42,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:01:45,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:49,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:01:50,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:51,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:01:55,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:01:58,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:01:58,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:01:58,400 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:02:01,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:02:01,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:06,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:11,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:02:12,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:13,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:02:13,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:02:13,398 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:02:17,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:19,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:02:21,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:02:22,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:27,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:02:28,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:02:28,494 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:02:31,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:02:33,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:38,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:41,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:02:43,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:02:43,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:02:43,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:02:43,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:48,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:49,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:02:51,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:02:53,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:02:58,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:02:58,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:02:58,493 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:02:59,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:01,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:03:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:11,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:03:13,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:03:13,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:03:13,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:03:15,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:19,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:03:20,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:21,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:03:25,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:28,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:03:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:03:28,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:03:31,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:31,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:03:36,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:41,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:42,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:03:43,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:03:43,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:03:43,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:03:46,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:49,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:03:51,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:03:51,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:56,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:03:58,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:03:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:03:58,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:03:59,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:04:02,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:07,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:08,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:04:08,467 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:04:08,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:04:08,470 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:04:09,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:04:09,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:04:13,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:13,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:04:13,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:04:13,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:04:18,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:19,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:04:19,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:04:23,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:04:28,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:04:28,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:04:29,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:04:29,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:34,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:40,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:41,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:04:43,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:04:43,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:04:43,493 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:04:45,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:49,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:04:50,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:51,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:04:55,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:04:58,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:04:58,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:04:58,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:05:01,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:05:01,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:06,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:11,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:05:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:05:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:05:13,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:05:17,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:19,915 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:05:21,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:05:22,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:27,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:29,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:05:29,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:05:29,064 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:05:31,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:05:33,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:38,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:41,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:05:43,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:44,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:05:44,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:05:44,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:05:49,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:49,916 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:05:51,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:05:54,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:05:59,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:05:59,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:05:59,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:05:59,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:01,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:06:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:10,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:12,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:06:14,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:06:14,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:06:14,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:06:15,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:19,917 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:06:20,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:22,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:06:25,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:29,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:06:29,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:06:29,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:06:30,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:32,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:06:35,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:40,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:42,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:06:44,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:06:44,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:06:44,122 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:06:46,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:49,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:06:51,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:53,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:06:56,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:06:59,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:06:59,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:06:59,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:07:02,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:03,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:07:07,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:12,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:13,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:07:14,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:07:14,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:07:14,122 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:07:18,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:19,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:07:23,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:07:23,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:28,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:07:30,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:07:30,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:07:31,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:07:31,638 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:07:31,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:07:31,640 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:07:32,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:07:33,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:07:34,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:39,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:43,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:07:45,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:45,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:07:45,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:07:45,574 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:07:49,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:07:50,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:07:51,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:07:55,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:08:00,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:08:00,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:08:01,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:01,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:08:06,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:11,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:11,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:08:15,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:08:15,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:08:15,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:08:16,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:19,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:08:22,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:22,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:08:27,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:30,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:08:30,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:08:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:08:32,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:08:32,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:37,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:42,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:08:43,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:45,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:08:45,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:08:45,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:08:48,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:49,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:08:52,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:08:53,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:08:58,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:09:00,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:09:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:09:02,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:09:04,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:09,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:10,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:09:14,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:15,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:09:15,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:09:15,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:09:19,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:09:19,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:20,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:09:24,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:29,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:30,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:09:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:09:30,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:09:30,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:09:35,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:39,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:09:40,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:45,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:09:45,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:09:45,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:09:46,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:49,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:09:49,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:09:51,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:56,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:09:59,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:10:00,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:10:00,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:10:00,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:10:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:07,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:15,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:10:15,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:10:15,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:10:18,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:19,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:10:23,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:28,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:30,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:10:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:10:30,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:10:34,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:36,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:10:39,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:45,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:45,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:10:45,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:10:45,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:10:46,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:10:49,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:10:50,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:10:56,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:10:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:11:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:11:00,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:11:01,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:07,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:08,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:11:12,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:15,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:11:15,418 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:11:15,418 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:11:15,421 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:11:15,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:11:15,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:11:15,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:11:16,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:11:17,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:18,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:11:19,935 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:11:22,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:28,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:11:28,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:11:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:11:30,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:11:33,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:36,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:11:39,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:44,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:45,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:11:45,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:11:45,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:11:48,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:11:49,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:49,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:11:54,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:11:58,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:12:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:12:00,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:12:00,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:12:05,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:08,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:12:11,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:15,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:12:15,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:12:15,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:12:17,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:18,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:12:19,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:12:22,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:27,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:12:28,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:30,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:12:30,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:12:30,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:12:34,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:37,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:12:40,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:45,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:12:45,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:12:45,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:12:47,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:12:49,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:12:50,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:55,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:12:57,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:13:00,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:13:00,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:13:00,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:13:01,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:07,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:08,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:13:12,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:15,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:13:15,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:13:15,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:13:17,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:18,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:13:19,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:13:22,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:28,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:13:28,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:30,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:13:30,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:13:30,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:13:33,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:38,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:13:39,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:44,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:45,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:13:45,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:13:45,617 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:13:48,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:13:49,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:13:49,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:54,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:13:58,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:00,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:14:00,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:14:00,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:14:00,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:06,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:06,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:11,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:15,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:14:15,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:14:15,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:14:16,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:16,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:19,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:14:21,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:26,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:26,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:30,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:14:30,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:14:30,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:14:32,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:34,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:14:34,706 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:14:34,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:14:34,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:14:34,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:14:35,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:36,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:38,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:43,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:45,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:14:45,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:14:45,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:14:46,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:48,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:49,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:14:54,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:14:54,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:14:59,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:01,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:15:02,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:15:02,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:15:04,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:15:05,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:10,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:14,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:15:15,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:16,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:15:16,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:15:16,995 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:15:19,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:15:20,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:24,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:15:26,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:31,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:32,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:15:32,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:15:32,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:15:34,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:15:37,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:42,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:44,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:15:47,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:15:47,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:15:47,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:15:47,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:49,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:15:53,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:15:54,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:15:58,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:02,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:16:02,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:16:02,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:16:03,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:04,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:16:08,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:12,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:16:13,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:17,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:16:17,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:16:17,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:16:19,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:19,932 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:16:22,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:16:25,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:30,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:32,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:16:32,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:16:32,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:16:32,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:16:35,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:40,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:45,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:46,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:16:47,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:16:47,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:16:47,254 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:16:49,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:16:50,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:56,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:16:56,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:01,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:02,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:17:02,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:17:02,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:17:04,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:06,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:11,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:17,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:17,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:17,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:17:17,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:17:17,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:17:19,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:17:22,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:27,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:28,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:17:32,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:17:32,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:17:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:34,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:38,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:44,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:44,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:47,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:17:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:17:47,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:17:49,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:17:49,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:17:52,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:17:52,656 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:17:52,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:17:52,658 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:17:53,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:17:54,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:17:55,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:00,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:02,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:18:02,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:18:02,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:18:04,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:18:05,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:10,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:14,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:18:16,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:17,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:18:17,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:18:17,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:18:19,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:18:21,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:22,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:18:26,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:31,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:32,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:18:32,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:18:32,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:18:32,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:18:36,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:42,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:42,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:18:47,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:18:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:18:47,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:18:49,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:18:52,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:18:52,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:18:57,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:00,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:02,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:19:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:19:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:02,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:19:06,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:08,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:13,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:14,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:17,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:19:17,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:19:17,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:19:18,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:19,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:19:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:24,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:19:32,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:19:32,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:19:35,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:35,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:40,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:45,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:46,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:47,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:19:47,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:19:47,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:19:49,940 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:19:51,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:19:55,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:19:57,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:20:02,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:20:02,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:02,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:20:03,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:20:07,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:12,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:20:13,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:17,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:20:17,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:20:17,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:20:18,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:19,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:20:22,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:20:24,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:29,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:32,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:20:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:20:32,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:20:32,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:20:34,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:39,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:40,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:20:45,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:48,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:20:48,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:20:48,620 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:20:49,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:20:50,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:20:50,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:20:55,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:00,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:21:00,023 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:21:00,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:21:00,025 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:21:00,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:21:00,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:01,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:03,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:21:03,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:21:03,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:21:06,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:10,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:12,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:17,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:21:18,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:21:18,620 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:21:19,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:21:20,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:22,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:27,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:30,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:33,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:33,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:21:33,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:21:33,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:21:38,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:38,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:43,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:48,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:21:48,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:21:48,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:21:48,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:48,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:49,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:21:53,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:21:59,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:21:59,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:03,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:22:03,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:22:03,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:22:04,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:09,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:22:09,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:14,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:22:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:22:18,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:22:19,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:22:19,944 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:22:19,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:24,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:27,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:22:30,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:34,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:22:34,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:22:34,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:22:36,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:37,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:22:41,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:46,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:47,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:22:49,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:22:49,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:22:49,104 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:22:49,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:22:51,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:22:57,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:22:57,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:02,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:23:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:23:04,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:23:05,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:23:08,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:13,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:14,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:23:18,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:19,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:23:19,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:23:19,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:23:19,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:23:24,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:24,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:23:29,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:34,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:23:34,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:23:34,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:23:34,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:23:35,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:40,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:44,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:23:45,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:49,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:23:49,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:23:49,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:23:49,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:23:50,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:23:52,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:23:56,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:01,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:02,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:24:04,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:24:04,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:24:04,150 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:24:06,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:11,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:24:11,006 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:24:11,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:24:11,008 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:24:11,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:24:12,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:13,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:24:17,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:19,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:24:20,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:24:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:24:20,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:24:22,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:23,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:24:27,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:33,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:33,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:24:35,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:24:35,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:24:35,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:24:38,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:43,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:24:43,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:48,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:49,949 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:24:50,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:24:50,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:24:50,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:24:53,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:24:53,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:24:58,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:03,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:25:04,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:25:05,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:25:05,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:25:10,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:11,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:25:15,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:19,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:25:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:25:20,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:25:20,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:25:20,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:23,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:25:25,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:31,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:33,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:25:35,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:25:35,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:25:35,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:25:37,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:41,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:25:43,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:48,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:49,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:25:50,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:25:50,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:25:50,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:25:51,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:25:53,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:25:58,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:00,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:26:04,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:26:05,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:26:05,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:26:09,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:10,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:26:14,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:19,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:26:19,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:26:20,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:26:20,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:26:20,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:26:25,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:29,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:26:30,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:35,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:36,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:26:36,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:26:36,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:26:39,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:26:40,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:45,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:49,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:26:49,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:26:50,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:26:51,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:26:51,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:26:51,726 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:26:56,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:01,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:06,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:08,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:27:08,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:27:08,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:27:11,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:27:12,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:17,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:19,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:27:21,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:27:22,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:23,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:27:23,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:27:23,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:27:28,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:31,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:27:34,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:38,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:27:38,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:27:38,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:27:39,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:39,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:27:39,317 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:27:39,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:27:39,353 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:27:39,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:27:40,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:27:41,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:27:44,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:49,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:27:49,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:27:51,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:27:53,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:27:53,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:27:53,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:27:55,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:01,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:01,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:28:06,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:28:08,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:28:08,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:28:11,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:11,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:28:16,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:18,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:28:19,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:28:21,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:23,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:28:23,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:28:23,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:28:27,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:29,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:28:32,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:38,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:28:38,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:38,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:28:38,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:28:39,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:28:43,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:48,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:49,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:28:49,954 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:28:53,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:28:53,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:28:53,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:28:54,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:28:59,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:00,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:05,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:07,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:08,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:29:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:29:08,179 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:29:10,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:15,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:17,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:19,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:29:20,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:24,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:29:24,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:29:24,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:29:26,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:27,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:36,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:37,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:39,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:29:39,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:29:39,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:29:41,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:47,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:47,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:49,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:29:52,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:29:54,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:29:54,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:29:54,942 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:29:57,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:29:57,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:02,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:07,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:08,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:09,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:30:09,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:30:09,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:30:14,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:17,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:19,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:30:19,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:24,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:30:24,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:30:24,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:30:25,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:25,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:30,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:35,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:35,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:39,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:30:39,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:30:39,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:30:41,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:46,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:46,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:49,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:30:51,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:30:54,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:30:54,310 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:30:54,311 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:30:54,313 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:30:54,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:30:54,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:30:54,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:30:55,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:30:55,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:56,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:30:57,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:02,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:06,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:31:07,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:09,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:31:09,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:31:09,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:31:13,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:18,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:19,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:31:19,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:31:23,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:26,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:31:26,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:31:26,249 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:31:29,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:29,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:31:34,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:37,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:31:39,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:41,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:31:41,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:31:41,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:31:45,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:49,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:31:49,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:31:50,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:55,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:31:56,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:31:56,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:31:56,249 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:31:57,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:01,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:07,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:07,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:11,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:32:11,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:32:11,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:32:12,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:17,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:17,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:19,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:32:22,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:26,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:32:26,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:32:26,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:32:27,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:28,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:33,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:37,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:39,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:41,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:32:41,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:32:41,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:32:44,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:45,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:49,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:49,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:32:55,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:32:56,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:32:56,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:32:56,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:32:56,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:33:00,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:05,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:06,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:33:10,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:11,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:33:11,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:33:11,985 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:33:14,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:33:15,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:19,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:33:21,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:26,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:33:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:33:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:33:26,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:33:27,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:32,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:34,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:33:38,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:41,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:33:41,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:33:41,986 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:33:43,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:33:43,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:48,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:49,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:33:53,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:33:53,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:33:56,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:33:56,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:33:56,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:33:59,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:03,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:34:04,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:09,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:11,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:34:11,409 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:34:11,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:34:11,412 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:34:11,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:34:11,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:34:11,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:34:11,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:34:13,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:34:15,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:19,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:34:20,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:21,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:34:25,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:34:26,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:34:26,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:34:31,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:31,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:34:36,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:41,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:34:41,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:41,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:34:41,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:34:41,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:34:47,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:49,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:34:49,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:34:52,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:57,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:34:58,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:34:58,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:34:58,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:35:01,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:35:03,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:08,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:10,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:35:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:35:13,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:35:13,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:35:13,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:18,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:19,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:35:20,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:35:23,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:28,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:35:28,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:35:28,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:35:29,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:30,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:35:34,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:39,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:40,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:35:43,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:35:43,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:35:43,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:35:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:49,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:35:50,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:35:50,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:55,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:35:58,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:35:58,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:35:58,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:36:00,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:36:01,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:06,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:08,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:36:11,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:36:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:36:13,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:36:17,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:18,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:36:19,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:36:22,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:28,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:28,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:36:28,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:36:28,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:36:28,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:36:33,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:38,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:36:38,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:36:43,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:36:43,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:36:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:49,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:36:49,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:36:49,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:54,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:36:58,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:36:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:36:58,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:37:00,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:05,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:07,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:37:11,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:13,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:37:13,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:37:13,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:37:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:17,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:37:19,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:37:21,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:27,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:28,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:37:28,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:37:28,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:37:28,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:37:33,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:36,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:37:36,657 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:37:36,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:37:36,660 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:37:37,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:37:38,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:37:38,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:37:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:37:43,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:37:43,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:48,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:37:49,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:49,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:37:54,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:37:58,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:37:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:37:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:37:58,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:38:00,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:06,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:06,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:38:11,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:13,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:38:13,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:38:13,573 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:38:16,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:38:16,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:19,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:38:21,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:26,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:38:27,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:28,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:38:28,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:38:28,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:38:33,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:34,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:38:38,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:43,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:38:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:38:43,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:38:46,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:38:48,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:49,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:38:53,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:38:54,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:38:58,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:38:58,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:38:58,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:38:59,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:03,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:39:05,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:10,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:13,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:39:13,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:39:13,572 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:39:13,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:39:15,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:19,968 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:39:20,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:24,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:39:26,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:28,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:39:28,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:39:28,572 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:39:31,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:34,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:39:37,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:42,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:39:43,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:39:43,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:39:44,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:39:48,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:49,969 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:39:52,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:39:54,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:39:58,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:39:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:39:58,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:39:59,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:02,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:05,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:10,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:12,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:13,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:40:13,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:40:13,572 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:40:15,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:19,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:40:21,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:22,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:26,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:29,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:40:29,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:40:29,723 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:40:31,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:32,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:36,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:42,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:42,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:44,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:40:44,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:40:44,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:40:47,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:49,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:40:49,847 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:40:49,847 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:40:49,849 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:40:49,971 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:40:50,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:40:50,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:51,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:40:52,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:57,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:40:59,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:40:59,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:40:59,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:41:00,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:09,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:10,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:14,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:14,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:41:14,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:41:14,719 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:41:19,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:41:19,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:20,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:24,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:29,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:41:29,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:41:29,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:41:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:30,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:34,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:39,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:40,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:44,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:41:44,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:41:44,724 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:41:45,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:48,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:49,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:41:50,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:55,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:41:58,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:41:59,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:41:59,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:41:59,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:42:01,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:07,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:08,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:42:12,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:15,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:42:15,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:42:15,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:42:18,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:18,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:42:19,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:42:23,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:28,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:42:29,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:30,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:42:30,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:42:30,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:42:34,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:38,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:42:40,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:45,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:42:45,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:42:45,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:42:46,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:42:49,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:42:50,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:56,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:42:56,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:00,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:43:00,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:43:00,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:43:01,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:06,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:07,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:12,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:15,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:43:15,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:43:15,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:43:16,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:17,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:19,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:43:22,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:26,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:28,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:30,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:43:30,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:43:30,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:43:34,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:34,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:39,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:44,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:44,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:43:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:43:45,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:43:49,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:43:49,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:43:54,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:43:55,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:00,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:01,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:44:01,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:44:01,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:44:02,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:44:02,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:44:02,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:44:02,679 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:44:02,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:44:03,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:05,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:05,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:10,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:15,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:16,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:16,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:44:16,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:44:16,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:44:19,977 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:44:21,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:22,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:26,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:31,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:44:31,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:44:31,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:44:31,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:32,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:36,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:42,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:42,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:46,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:44:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:44:46,438 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:44:47,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:49,978 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:44:52,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:44:53,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:44:58,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:01,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:45:01,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:45:01,438 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:45:02,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:45:03,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:08,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:12,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:45:13,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:16,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:45:16,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:45:16,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:45:19,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:19,978 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:45:20,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:45:24,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:30,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:45:30,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:31,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:45:31,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:45:31,438 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:45:36,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:40,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:45:42,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:45:46,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:45:46,439 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:45:47,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:49,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:45:52,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:45:54,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:45:57,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:02,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:46:02,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:46:02,129 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:46:03,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:04,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:46:08,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:13,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:14,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:46:17,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:46:17,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:46:17,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:46:19,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:19,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:46:24,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:46:25,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:30,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:32,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:46:32,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:46:32,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:46:33,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:46:35,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:40,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:43,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:46:45,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:47,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:46:47,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:46:47,129 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:46:49,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:46:50,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:46:51,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:46:55,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:01,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:02,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:47:02,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:47:02,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:47:03,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:07,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:11,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:47:17,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:47:17,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:47:18,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:18,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:47:18,677 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:47:18,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:47:18,680 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:47:19,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:47:19,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:47:20,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:21,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:23,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:29,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:30,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:32,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:47:32,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:47:32,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:47:34,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:38,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:40,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:45,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:47,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:47:47,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:47:47,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:47:48,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:47:49,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:47:50,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:55,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:47:58,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:48:01,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:02,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:48:02,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:48:02,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:48:07,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:08,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:48:12,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:17,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:48:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:48:17,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:48:18,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:48:19,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:48:22,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:28,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:28,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:48:32,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:48:32,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:48:32,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:48:33,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:38,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:38,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:48:43,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:47,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:48:47,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:48:47,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:48:49,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:48:49,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:49,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:48:54,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:48:57,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:00,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:02,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:49:02,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:49:02,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:49:06,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:07,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:11,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:17,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:49:17,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:49:17,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:49:19,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:49:21,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:27,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:27,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:32,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:49:33,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:49:33,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:49:37,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:38,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:43,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:45,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:48,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:49:48,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:49:48,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:49:48,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:49,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:49:53,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:49:55,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:49:58,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:02,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:50:03,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:50:03,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:50:03,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:50:04,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:09,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:12,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:50:14,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:18,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:50:18,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:50:18,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:50:19,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:50:19,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:22,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:50:25,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:30,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:31,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:50:31,483 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:50:31,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:50:31,486 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:50:31,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:50:32,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:50:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:50:33,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:50:33,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:50:36,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:41,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:42,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:50:46,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:49,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:50:49,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:50:49,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:50:49,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:50:52,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:50:52,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:50:57,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:00,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:51:03,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:04,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:51:04,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:51:04,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:51:08,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:11,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:51:14,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:19,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:51:19,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:51:19,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:51:19,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:19,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:51:21,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:51:24,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:30,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:31,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:51:34,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:51:34,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:51:34,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:51:36,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:41,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:51:42,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:47,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:49,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:51:49,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:51:49,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:51:49,437 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:51:49,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:51:52,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:58,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:51:59,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:03,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:05,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:52:05,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:52:05,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:52:08,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:09,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:13,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:17,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:18,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:19,989 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:52:20,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:52:20,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:52:20,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:52:24,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:27,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:30,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:35,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:52:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:52:35,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:52:37,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:40,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:46,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:46,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:50,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:52:50,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:52:50,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:52:50,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:52:51,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:52:54,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:52:57,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:02,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:04,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:05,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:53:05,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:53:05,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:53:08,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:14,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:14,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:19,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:19,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:53:20,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:53:20,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:53:20,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:53:25,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:25,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:30,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:35,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:53:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:53:35,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:53:35,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:40,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:42,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:53:42,217 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:53:42,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:53:42,218 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:53:43,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:53:43,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:45,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:46,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:49,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:53:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:53:51,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:53:51,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:53:51,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:53:55,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:53:57,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:02,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:03,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:54:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:54:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:54:06,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:54:08,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:13,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:54:14,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:19,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:19,993 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:54:21,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:54:21,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:54:21,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:54:23,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:54:24,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:29,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:33,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:54:35,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:36,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:54:36,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:54:36,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:54:40,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:43,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:54:45,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:49,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:54:50,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:54:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:54:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:54:51,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:54:53,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:54:56,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:01,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:55:02,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:06,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:55:06,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:55:06,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:55:07,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:11,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:55:13,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:18,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:19,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:55:21,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:55:21,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:55:21,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:55:21,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:55:23,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:28,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:30,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:55:34,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:36,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:55:36,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:55:36,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:55:39,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:41,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:55:44,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:49,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:49,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:55:51,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:55:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:55:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:55:51,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:55:55,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:55:59,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:01,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:06,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:06,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:56:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:56:06,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:56:09,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:11,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:16,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:19,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:19,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:56:21,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:56:21,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:56:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:56:21,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:26,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:29,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:32,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:36,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:56:36,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:56:36,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:56:37,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:37,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:42,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:47,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:48,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:49,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:56:51,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:56:51,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:56:51,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:56:53,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:56:54,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 16:56:54,976 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 16:56:54,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 16:56:54,982 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 16:56:55,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 16:56:56,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:57,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:56:58,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:03,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:05,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:57:06,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:57:06,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:57:06,550 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:57:09,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:15,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:57:15,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:19,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:57:20,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:23,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:57:23,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:57:23,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:57:25,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:57:26,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:31,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:35,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:57:36,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:38,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:57:38,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:57:38,055 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:57:42,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:45,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:57:47,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:49,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:57:53,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:57:53,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:57:53,055 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:57:53,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:57:53,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:57:58,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:03,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:04,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:58:08,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:58:08,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:58:08,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:58:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:14,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:58:14,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:19,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:20,001 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:58:23,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:58:23,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:58:23,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:58:23,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:58:25,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:30,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:33,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:58:35,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:38,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:58:38,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:58:38,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:58:41,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:43,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:58:46,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:50,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:58:51,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:58:51,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:58:53,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:58:53,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:58:53,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:58:57,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:01,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:59:02,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:07,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:09,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:59:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:59:09,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:59:11,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:59:13,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:18,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:20,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:59:21,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:59:24,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:24,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:59:24,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:59:24,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:59:29,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:31,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:59:34,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:39,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:39,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:59:39,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:59:39,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:59:41,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:59:44,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:49,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:50,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 16:59:54,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 16:59:54,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 16:59:54,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 16:59:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 16:59:55,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 16:59:59,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:04,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:05,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:09,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:00:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:00:09,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:00:10,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:13,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:00:13,286 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:00:13,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:00:13,289 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:00:13,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:00:14,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:15,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:16,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:20,005 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:00:22,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:24,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:00:24,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:00:24,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:00:26,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:27,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:32,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:34,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:37,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:39,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:00:39,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:00:39,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:00:43,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:44,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:48,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:50,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:00:53,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:00:54,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:00:54,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:00:54,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:00:54,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:00:58,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:03,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:04,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:01:08,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:01:10,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:01:10,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:01:13,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:01:14,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:19,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:20,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:01:21,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:01:24,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:25,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:01:25,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:01:25,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:01:29,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:31,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:01:34,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:39,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:40,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:01:40,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:01:40,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:01:41,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:01:45,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:50,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:01:51,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:51,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:01:55,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:01:55,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:01:55,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:01:56,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:01:59,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:02:01,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:06,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:09,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:02:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:02:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:02:10,103 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:02:12,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:17,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:19,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:02:20,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:02:23,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:26,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:02:26,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:02:26,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:02:28,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:29,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:02:33,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:38,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:39,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:02:41,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:02:41,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:02:41,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:02:44,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:48,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:02:49,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:50,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:02:55,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:02:56,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:02:56,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:02:56,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:02:58,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:00,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:05,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:08,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:11,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:11,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:03:11,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:03:11,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:03:16,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:18,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:20,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:03:22,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:26,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:03:26,082 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:03:26,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:03:26,085 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:03:26,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:03:26,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:03:26,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:03:26,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:03:27,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:27,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:28,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:32,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:38,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:41,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:03:41,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:03:41,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:03:43,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:48,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:03:49,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:50,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:03:54,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:03:56,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:03:56,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:03:56,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:03:56,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:00,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:05,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:06,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:11,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:04:12,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:04:12,534 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:04:15,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:16,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:20,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:04:22,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:25,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:27,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:27,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:04:27,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:04:27,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:04:32,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:35,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:38,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:04:42,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:04:42,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:04:43,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:43,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:48,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:50,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:04:53,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:04:54,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:04:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:04:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:04:57,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:04:59,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:04,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:05:05,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:10,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:12,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:05:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:05:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:05:12,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:05:15,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:20,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:05:21,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:22,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:05:26,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:05:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:05:27,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:05:31,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:32,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:05:36,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:42,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:05:42,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:05:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:05:42,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:05:47,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:50,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:05:52,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:05:53,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:05:57,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:05:57,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:05:57,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:05:58,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:02,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:04,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:09,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:12,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:06:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:06:12,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:06:14,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:20,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:06:20,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:22,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:25,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:06:27,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:06:27,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:06:30,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:30,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:35,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:38,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:06:38,404 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:06:38,405 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:06:38,407 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:06:38,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:06:39,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:40,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:41,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:42,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:06:42,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:06:42,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:06:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:49,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:50,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:06:53,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:06:57,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:06:57,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:06:57,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:06:57,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:06:58,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:03,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:07,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:07:08,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:12,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:07:12,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:07:12,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:07:14,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:17,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:07:19,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:20,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:07:25,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:07:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:07:27,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:07:28,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:07:30,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:35,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:38,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:07:40,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:43,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:07:43,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:07:43,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:07:46,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:46,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:07:50,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:07:52,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:56,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:07:57,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:07:58,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:07:58,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:07:58,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:08:03,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:06,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:08:08,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:13,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:08:13,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:08:13,928 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:08:14,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:16,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:08:19,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:20,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:08:25,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:26,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:08:28,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:08:28,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:08:28,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:08:30,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:35,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:36,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:08:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:43,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:08:43,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:08:43,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:08:45,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:46,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:08:50,019 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:08:51,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:56,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:08:56,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:08:58,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:08:58,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:08:58,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:09:02,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:03,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:07,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:12,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:13,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:09:13,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:09:13,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:09:14,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:18,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:20,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:09:23,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:24,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:28,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:28,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:09:28,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:09:28,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:09:34,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:34,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:39,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:43,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:09:43,928 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:09:43,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:09:44,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:45,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:50,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:09:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:09:52,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:09:52,116 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:09:52,117 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:09:52,118 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:09:52,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:09:53,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:54,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:09:56,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:10:00,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:10:00,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:10:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:02,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:10:06,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:11,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:12,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:10:15,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:10:15,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:10:15,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:10:17,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:20,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:10:22,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:10:23,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:30,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:10:30,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:10:30,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:10:32,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:10:33,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:38,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:42,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:10:44,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:45,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:10:45,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:10:45,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:10:49,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:10:50,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:10:52,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:10:55,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:00,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:00,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:11:00,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:11:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:11:00,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:05,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:10,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:10,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:15,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:11:15,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:11:15,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:11:16,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:20,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:11:21,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:22,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:27,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:30,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:11:30,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:11:30,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:11:31,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:32,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:37,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:40,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:43,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:11:46,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:11:46,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:11:48,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:48,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:50,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:11:54,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:11:58,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:11:59,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:12:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:12:01,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:12:05,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:08,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:12:11,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:16,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:16,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:12:16,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:12:16,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:12:18,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:12:20,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:12:22,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:27,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:28,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:12:31,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:12:31,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:12:31,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:12:32,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:38,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:38,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:12:43,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:12:46,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:12:46,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:12:48,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:12:48,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:50,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:12:54,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:12:56,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:12:59,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:13:01,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:13:01,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:13:04,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:05,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:13:05,766 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:13:05,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:13:05,768 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:13:06,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:13:06,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:07,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:10,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:15,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:16,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:13:16,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:13:16,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:13:16,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:20,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:13:21,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:26,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:26,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:32,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:13:32,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:13:32,708 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:13:34,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:36,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:41,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:45,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:47,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:47,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:13:47,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:13:47,706 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:13:50,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:13:52,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:13:55,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:13:57,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:02,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:02,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:14:02,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:14:02,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:14:08,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:09,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:14:13,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:17,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:14:17,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:14:17,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:14:18,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:14:18,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:20,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:14:24,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:26,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:14:29,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:34,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:14:34,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:14:34,155 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:14:35,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:36,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:14:40,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:45,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:49,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:14:49,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:14:49,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:14:50,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:14:51,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:14:56,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:01,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:02,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:15:04,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:15:04,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:15:04,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:15:07,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:10,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:15:12,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:17,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:19,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:15:19,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:15:19,153 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:15:20,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:15:20,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:15:23,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:28,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:30,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:15:33,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:34,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:15:34,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:15:34,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:15:39,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:40,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:15:44,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:49,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:15:49,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:15:49,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:15:49,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:15:49,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:50,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:15:55,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:15:59,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:00,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:04,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:16:04,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:16:04,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:16:06,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:09,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:11,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:16,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:19,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:16:19,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:16:19,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:16:19,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:20,030 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:16:22,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:27,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:27,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:32,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:16:34,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:16:34,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:16:35,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:16:35,566 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:16:35,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:16:35,569 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:16:36,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:16:37,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:37,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:38,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:42,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:46,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:48,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:49,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:16:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:16:49,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:16:50,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:16:54,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:16:56,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:16:59,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:04,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:04,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:17:04,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:17:04,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:17:06,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:17:09,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:14,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:17:15,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:19,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:17:19,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:17:19,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:17:20,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:17:21,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:24,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:17:26,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:31,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:17:34,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:17:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:17:34,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:17:36,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:41,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:44,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:17:46,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:17:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:17:49,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:17:50,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:17:52,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:17:55,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:17:58,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:03,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:18:03,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:04,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:18:04,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:18:04,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:18:08,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:13,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:18:14,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:19,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:20,034 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:18:20,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:18:20,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:18:20,172 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:18:23,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:18:25,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:30,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:33,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:18:35,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:18:35,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:18:35,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:18:35,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:40,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:43,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:18:45,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:50,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:18:50,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:18:50,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:18:50,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:18:51,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:18:53,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:18:56,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:01,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:01,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:19:05,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:19:05,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:19:05,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:19:06,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:10,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:19:12,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:17,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:20,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:19:20,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:19:20,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:19:20,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:19:20,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:19:22,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:27,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:30,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:19:32,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:35,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:19:35,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:19:35,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:19:38,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:40,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:19:44,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:48,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:19:48,744 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:19:48,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:19:48,747 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:19:48,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:19:49,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:50,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:19:50,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:19:50,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:19:50,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:19:50,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:19:55,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:19:59,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:00,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:05,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:06,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:20:06,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:20:06,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:20:09,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:11,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:16,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:19,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:20,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:20:21,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:20:21,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:21,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:20:21,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:20:26,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:29,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:32,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:36,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:20:36,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:20:36,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:20:37,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:37,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:42,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:47,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:48,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:50,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:20:51,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:20:51,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:20:51,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:20:53,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:20:57,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:20:58,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:03,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:06,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:21:06,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:21:06,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:21:07,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:21:09,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:14,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:17,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:21:20,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:21:20,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:21,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:21:21,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:21:21,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:21:25,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:25,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:21:30,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:34,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:21:35,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:36,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:21:36,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:21:36,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:21:41,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:44,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:21:47,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:50,038 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:21:52,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:21:52,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:21:52,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:21:52,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:21:55,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:21:58,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:03,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:22:03,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:22:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:22:07,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:22:09,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:13,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:22:15,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:20,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:22:21,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:22:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:22:22,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:22:23,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:22:26,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:31,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:33,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:22:36,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:37,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:22:37,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:22:37,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:22:41,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:22:42,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:47,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:50,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:22:51,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:22:52,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:22:52,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:22:52,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:22:53,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:22:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:00,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:23:00,406 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:23:00,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:23:00,411 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:23:00,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:23:01,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:02,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:04,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:23:08,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:23:08,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:23:09,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:11,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:14,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:20,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:23:20,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:21,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:23:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:23:23,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:23:25,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:29,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:31,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:36,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:38,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:23:38,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:23:38,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:23:39,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:41,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:46,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:49,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:23:50,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:23:52,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:53,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:23:53,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:23:53,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:23:58,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:23:59,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:24:03,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:08,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:08,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:24:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:24:08,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:24:09,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:24:13,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:17,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:24:19,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:20,044 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:24:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:24:23,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:24:23,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:24:25,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:27,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:24:30,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:35,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:38,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:24:38,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:24:38,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:24:39,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:24:40,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:45,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:49,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:24:50,043 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:24:51,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:53,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:24:53,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:24:53,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:24:56,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:24:57,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:01,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:07,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:07,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:25:08,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:25:08,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:25:12,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:13,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:14,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:17,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:20,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:25:22,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:23,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:23,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:25:23,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:25:23,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:25:28,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:33,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:35,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:38,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:38,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:25:38,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:25:38,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:25:43,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:44,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:49,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:50,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:25:53,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:25:53,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:25:53,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:25:53,787 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:25:54,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:25:59,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:03,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:05,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:08,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:26:08,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:26:08,788 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:26:10,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:11,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:26:11,576 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:26:11,585 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:26:11,587 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:26:11,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:26:12,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:13,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:16,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:20,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:26:21,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:21,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:23,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:26:23,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:26:23,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:26:26,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:31,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:32,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:36,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:39,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:26:39,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:26:39,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:26:42,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:42,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:47,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:50,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:26:52,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:26:53,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:26:54,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:26:54,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:26:54,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:26:58,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:00,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:27:04,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:09,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:27:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:27:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:27:09,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:09,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:27:14,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:19,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:27:20,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:27:20,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:24,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:27:24,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:27:24,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:27:25,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:29,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:27:30,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:35,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:37,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:27:39,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:27:39,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:27:39,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:27:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:46,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:47,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:27:50,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:27:52,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:54,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:27:54,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:27:54,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:27:57,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:27:59,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:28:02,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:07,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:28:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:28:09,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:28:09,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:28:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:17,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:28:19,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:20,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:28:25,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:25,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:28:25,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:28:25,896 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:28:29,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:28:30,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:35,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:28:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:28:40,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:28:41,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:43,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:28:46,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:50,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:28:51,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:28:53,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:28:55,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:28:55,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:28:55,893 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:28:57,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:02,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:02,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:07,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:10,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:29:10,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:29:10,893 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:29:12,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:13,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:18,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:20,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:29:22,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:24,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:25,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:29:25,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:29:25,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:29:29,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:29,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:29:29,166 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:29:29,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:29:29,168 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:29:29,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:29:31,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:32,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:34,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:39,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:40,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:29:40,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:29:40,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:29:45,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:50,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:29:50,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:29:51,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:56,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:29:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:29:56,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:29:56,114 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:30:00,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:30:01,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:06,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:09,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:30:11,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:30:11,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:30:11,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:30:11,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:16,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:19,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:30:20,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:30:22,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:30:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:30:26,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:30:27,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:27,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:30:32,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:37,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:30:38,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:41,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:30:41,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:30:41,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:30:43,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:47,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:30:48,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:50,066 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:30:54,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:30:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:30:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:30:56,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:30:59,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:04,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:07,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:31:09,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:11,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:31:11,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:31:11,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:31:15,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:18,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:31:20,066 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:31:21,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:26,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:31:26,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:31:26,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:26,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:31:28,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:31:31,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:36,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:31:36,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:41,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:31:41,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:31:41,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:31:42,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:46,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:31:47,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:50,067 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:31:53,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:31:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:31:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:31:56,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:31:56,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:31:58,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:03,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:06,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:12,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:32:12,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:32:12,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:32:14,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:16,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:19,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:20,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:32:24,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:24,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:32:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:32:27,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:32:29,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:34,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:34,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:39,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:42,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:32:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:32:42,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:32:44,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:45,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:50,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:32:51,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:51,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:32:51,486 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:32:51,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:32:51,488 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:32:51,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:32:53,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:54,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:32:56,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:32:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:32:57,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:32:57,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:33:01,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:33:01,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:06,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:11,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:33:12,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:33:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:33:12,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:33:17,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:20,070 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:33:21,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:33:23,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:27,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:33:27,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:33:27,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:33:28,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:31,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:33:34,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:39,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:42,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:33:42,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:33:42,551 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:33:42,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:33:44,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:50,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:33:50,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:52,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:33:55,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:33:57,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:33:57,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:33:57,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:34:00,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:00,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:05,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:10,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:12,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:34:12,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:34:12,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:34:16,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:20,072 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:34:20,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:22,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:27,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:34:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:34:27,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:34:30,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:32,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:37,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:40,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:42,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:42,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:34:42,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:34:42,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:34:48,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:48,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:50,072 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:34:53,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:34:57,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:34:57,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:34:57,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:34:58,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:34:59,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:04,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:08,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:35:09,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:12,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:35:12,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:35:12,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:35:15,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:18,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:35:20,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:35:20,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:25,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:27,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:35:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:35:27,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:35:28,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:35:31,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:36,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:36,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:35:41,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:43,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:35:43,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:35:43,024 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:35:45,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:35:47,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:50,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:35:52,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:53,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:35:57,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:35:58,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:35:58,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:35:58,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:36:02,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:02,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:36:02,956 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:36:02,957 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:36:02,958 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:36:03,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:36:04,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:05,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:07,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:13,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:36:13,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:36:13,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:36:13,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:14,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:18,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:20,076 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:36:24,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:24,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:28,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:36:28,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:36:28,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:36:29,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:34,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:34,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:39,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:43,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:36:43,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:36:43,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:36:44,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:45,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:50,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:36:51,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:54,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:36:56,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:36:58,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:36:58,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:36:58,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:37:01,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:02,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:37:06,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:11,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:12,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:37:13,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:37:13,022 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:37:13,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:37:17,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:20,078 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:37:22,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:37:23,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:28,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:28,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:37:28,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:37:28,501 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:37:32,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:37:33,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:38,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:40,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:37:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:37:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:37:43,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:37:44,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:50,078 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:37:50,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:50,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:37:55,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:37:58,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:37:58,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:37:58,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:38:00,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:00,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:38:05,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:11,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:38:11,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:38:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:38:13,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:38:16,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:19,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:38:20,081 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:38:22,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:27,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:28,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:38:28,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:38:28,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:38:28,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:38:32,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:36,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:38:38,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:43,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:45,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:38:45,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:38:45,036 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:38:49,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:38:49,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:50,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:38:55,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:38:56,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:00,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:39:00,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:39:00,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:39:00,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:05,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:06,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:10,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:14,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:39:14,901 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:39:14,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:39:14,903 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:39:15,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:39:15,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:39:15,041 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:39:15,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:39:16,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:16,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:20,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:39:22,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:26,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:27,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:30,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:39:30,044 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:39:30,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:39:33,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:34,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:38,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:43,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:44,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:45,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:39:45,041 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:39:45,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:39:49,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:50,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:39:54,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:39:54,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:39:59,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:40:01,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:40:01,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:40:04,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:04,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:40:09,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:12,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:40:15,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:16,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:40:16,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:40:16,623 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:40:20,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:40:20,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:22,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:40:25,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:30,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:31,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:40:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:40:31,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:40:32,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:40:35,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:40,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:42,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:40:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:46,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:40:46,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:40:46,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:40:50,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:40:52,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:40:53,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:40:57,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:41:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:41:01,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:41:02,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:03,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:41:07,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:12,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:13,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:41:16,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:41:16,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:41:16,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:41:17,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:20,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:41:21,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:41:23,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:28,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:41:31,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:41:31,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:41:33,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:41:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:38,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:41,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:41:44,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:46,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:41:46,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:41:46,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:41:49,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:41:50,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:41:50,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:41:55,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:00,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:00,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:42:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:42:01,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:42:01,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:42:05,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:10,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:42:10,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:15,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:42:18,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:42:18,135 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:42:20,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:42:20,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:42:21,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:26,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:28,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:42:28,981 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:42:28,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:42:28,984 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:42:29,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:42:30,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:42:31,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:42:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:42:33,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:42:37,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:40,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:42:42,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:47,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:48,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:42:48,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:42:48,133 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:42:48,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:42:50,085 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:42:53,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:42:58,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:02,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:43:03,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:43:03,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:43:03,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:43:03,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:08,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:12,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:43:14,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:43:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:43:18,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:43:19,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:20,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:43:23,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:43:24,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:29,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:33,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:43:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:43:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:43:33,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:43:35,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:40,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:43,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:43:45,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:48,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:43:48,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:43:48,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:43:50,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:43:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:43:51,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:43:56,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:01,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:01,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:44:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:44:03,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:44:07,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:11,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:12,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:17,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:18,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:44:18,948 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:44:18,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:44:20,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:44:21,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:23,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:28,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:31,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:33,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:33,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:44:33,948 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:44:33,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:44:38,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:38,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:43,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:48,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:48,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:48,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:44:48,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:44:48,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:44:50,088 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:44:54,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:44:58,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:44:59,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:03,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:45:03,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:45:03,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:45:05,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:08,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:45:11,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:16,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:18,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:45:18,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:45:18,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:45:18,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:45:20,088 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:45:21,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:26,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:28,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:45:31,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:33,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:45:33,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:45:33,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:45:37,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:39,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:45:42,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:46,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:45:46,750 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:45:46,750 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:45:46,753 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:45:47,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:45:47,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:48,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:45:48,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:45:48,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:45:49,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:45:50,089 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:45:53,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:58,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:45:59,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:46:03,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:04,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:46:04,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:46:04,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:46:07,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:46:08,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:13,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:17,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:46:19,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:19,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:46:19,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:46:19,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:46:20,089 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:46:24,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:27,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:46:29,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:34,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:34,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:46:34,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:46:34,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:46:37,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:46:39,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:45,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:45,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:46:49,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:46:49,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:46:49,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:46:50,090 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:46:50,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:55,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:46:55,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:00,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:04,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:47:04,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:47:04,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:47:05,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:05,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:10,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:14,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:16,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:20,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:47:21,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:47:21,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:47:21,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:47:22,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:22,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:27,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:32,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:34,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:36,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:47:36,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:47:36,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:47:37,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:42,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:43,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:47,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:50,093 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:47:51,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:47:51,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:47:51,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:47:52,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:47:53,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:47:57,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:02,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:03,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:48:06,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:48:06,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:48:06,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:48:08,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:13,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:48:13,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:18,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:20,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:48:21,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:48:21,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:48:21,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:48:23,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:48:24,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:29,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:33,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:48:35,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:36,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:48:36,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:48:36,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:48:40,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:43,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:48:45,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:50,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:48:51,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:51,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:48:51,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:48:51,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:48:53,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:48:56,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:48:59,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:48:59,874 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:48:59,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:48:59,877 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:49:00,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:49:01,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:49:01,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:06,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:49:06,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:49:06,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:49:07,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:11,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:49:12,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:17,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:20,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:49:21,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:49:21,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:49:21,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:49:21,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:49:23,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:28,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:30,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:49:33,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:36,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:49:36,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:49:36,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:49:38,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:40,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:49:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:49,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:49:49,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:50,096 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:49:51,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:49:51,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:49:51,043 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:49:55,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:49:59,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:50:01,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:06,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:50:06,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:50:06,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:50:09,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:50:11,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:16,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:19,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:50:20,096 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:50:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:50:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:50:21,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:50:22,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:27,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:29,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:50:32,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:36,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:50:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:50:36,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:50:38,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:39,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:50:43,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:49,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:49,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:50:50,108 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:50:51,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:50:51,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:50:51,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:50:54,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:50:57,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:00,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:05,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:51:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:51:06,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:51:07,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:10,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:15,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:17,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:20,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:51:21,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:21,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:51:21,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:51:21,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:51:26,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:27,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:31,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:36,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:51:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:51:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:36,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:51:37,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:41,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:47,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:47,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:50,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:51:51,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:51:51,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:51:51,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:51:52,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:51:55,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:51:58,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:03,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:06,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:52:06,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:52:06,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:52:08,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:08,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:13,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:14,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:52:14,681 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:52:14,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:52:14,684 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:52:15,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:52:15,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:18,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:19,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:20,098 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:52:21,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:52:21,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:52:21,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:52:25,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:26,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:30,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:35,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:35,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:52:36,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:52:36,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:52:41,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:45,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:47,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:50,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:52:52,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:52:52,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:52:52,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:52:52,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:52:55,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:52:58,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:03,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:05,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:53:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:53:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:53:07,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:53:09,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:13,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:53:15,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:20,100 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:53:21,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:53:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:53:22,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:53:23,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:53:26,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:32,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:33,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:53:37,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:53:37,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:53:37,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:53:38,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:43,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:53:44,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:49,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:50,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:53:52,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:53:52,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:53:52,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:53:53,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:53:54,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:53:59,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:03,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:54:05,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:54:07,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:54:07,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:54:10,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:11,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:54:15,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:20,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:54:21,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:21,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:54:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:54:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:54:22,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:54:26,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:32,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:54:32,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:37,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:38,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:54:38,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:54:38,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:54:42,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:54:42,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:47,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:50,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:54:52,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:54:53,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:54:53,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:54:53,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:54:53,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:54:58,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:02,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:55:04,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:08,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:55:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:55:08,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:55:09,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:10,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:55:14,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:19,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:55:19,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:20,103 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:55:23,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:55:23,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:55:23,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:55:25,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:27,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:55:27,316 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:55:27,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:55:27,319 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:55:27,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:55:29,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:55:31,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:36,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:38,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:55:38,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:55:38,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:55:39,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:55:41,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:46,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:49,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:55:50,104 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:55:52,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:55:53,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:55:53,631 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:55:57,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:55:57,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:02,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:07,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:56:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:56:08,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:56:12,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:17,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:18,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:20,104 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:56:23,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:56:23,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:56:23,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:56:23,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:25,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:29,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:35,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:35,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:38,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:56:38,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:56:38,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:56:40,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:46,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:46,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:50,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:56:52,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:56:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:56:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:56:53,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:56:56,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:56:57,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:02,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:06,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:57:07,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:08,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:57:08,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:57:08,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:57:13,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:19,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:20,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:57:20,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:57:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:57:23,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:57:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:57:24,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:29,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:34,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:38,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:57:38,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:57:38,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:57:39,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:44,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:49,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:57:50,108 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:57:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:57:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:57:53,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:57:54,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:57:55,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:00,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:03,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:58:06,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:08,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:58:08,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:58:08,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:58:11,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:13,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:58:17,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:20,109 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:58:22,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:58:23,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:58:23,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:58:23,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:58:27,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:32,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:33,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:58:37,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:38,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:58:38,632 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:58:38,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:58:43,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:43,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:58:48,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:50,110 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:58:51,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:58:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:58:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:58:53,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:58:58,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:01,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:59:04,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:09,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:09,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:59:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:59:09,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:59:10,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 17:59:10,706 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 17:59:10,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 17:59:10,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 17:59:11,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 17:59:12,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:59:14,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:19,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:20,110 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:59:22,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:59:24,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:59:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:59:24,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:59:24,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:29,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:32,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:59:35,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:39,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:59:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:59:39,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:59:40,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:42,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:59:45,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:50,111 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 17:59:51,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 17:59:52,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 17:59:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 17:59:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 17:59:54,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 17:59:56,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:02,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:00:02,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:07,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:09,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:00:09,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:00:09,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:00:10,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:00:12,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:17,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:20,111 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:00:20,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:00:23,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:00:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:00:24,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:00:28,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:30,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:00:33,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:39,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:00:39,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:00:39,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:00:40,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:00:44,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:50,112 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:00:50,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:50,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:00:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:00:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:00:54,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:00:55,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:00:59,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:01,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:06,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:07,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:01:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:01:09,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:01:11,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:16,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:17,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:20,112 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:01:22,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:01:25,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:01:25,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:01:27,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:27,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:32,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:38,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:38,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:40,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:01:40,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:01:40,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:01:43,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:48,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:48,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:50,114 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:01:54,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:01:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:01:55,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:01:55,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:01:58,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:01:59,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:04,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:08,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:02:09,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:10,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:02:10,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:02:10,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:02:15,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:16,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:02:20,116 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:02:21,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:25,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:02:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:02:25,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:02:25,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:02:25,757 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:02:25,757 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:02:25,760 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:02:26,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:02:26,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:28,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:02:31,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:36,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:38,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:02:40,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:02:40,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:02:40,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:02:42,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:46,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:02:48,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:50,115 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:02:54,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:02:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:02:55,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:02:55,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:02:56,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:02:59,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:05,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:06,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:03:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:03:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:03:10,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:03:10,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:16,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:16,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:03:20,116 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:03:22,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:03:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:03:25,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:03:26,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:03:27,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:32,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:34,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:03:37,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:03:40,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:03:40,351 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:03:43,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:43,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:03:48,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:50,119 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:03:53,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:03:54,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:03:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:03:55,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:03:55,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:03:58,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:04,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:04:04,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:09,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:10,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:04:10,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:04:10,796 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:04:14,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:04:15,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:20,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:20,120 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:04:22,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:04:25,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:25,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:04:25,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:04:25,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:04:30,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:32,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:04:35,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:40,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:04:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:04:40,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:04:41,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:42,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:04:46,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:50,120 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:04:50,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:04:51,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:04:55,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:04:55,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:04:55,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:04:56,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:02,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:02,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:07,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:10,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:10,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:05:10,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:05:10,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:05:13,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:18,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:20,121 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:05:20,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:23,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:25,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:05:25,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:05:25,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:05:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:30,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:38,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:05:38,470 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:05:38,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:05:38,474 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:05:38,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:05:39,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:39,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:05:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:05:40,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:05:40,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:45,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:50,121 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:05:50,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:50,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:05:55,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:05:55,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:05:55,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:05:55,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:06:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:03,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:06:05,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:10,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:06:10,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:06:10,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:06:11,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:11,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:06:16,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:20,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:06:21,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:21,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:06:26,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:27,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:06:27,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:06:27,047 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:06:30,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:06:31,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:36,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:38,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:06:41,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:42,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:06:42,045 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:06:42,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:06:47,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:48,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:06:50,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:06:53,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:57,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:06:57,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:06:57,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:06:58,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:06:58,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:03,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:08,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:08,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:12,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:07:12,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:07:12,045 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:07:14,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:18,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:19,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:20,134 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:07:25,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:27,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:07:27,045 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:07:27,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:07:28,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:30,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:35,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:38,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:40,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:42,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:07:42,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:07:42,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:07:45,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:46,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:50,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:07:51,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:56,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:07:57,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:07:57,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:07:57,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:07:57,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:08:02,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:07,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:08:08,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:13,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:08:13,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:08:13,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:08:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:08:18,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:20,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:08:24,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:27,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:08:28,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:08:28,348 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:08:28,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:08:29,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:36,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:08:40,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:08:43,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:08:43,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:08:46,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:46,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:08:50,125 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:08:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:53,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:08:53,400 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:08:53,401 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:08:53,403 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:08:53,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:08:54,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:08:57,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:08:58,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:08:58,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:08:58,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:09:02,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:04,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:09:07,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:13,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:13,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:09:13,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:09:13,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:09:14,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:09:18,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:20,128 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:09:24,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:24,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:09:28,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:09:28,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:09:28,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:09:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:34,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:09:34,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:39,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:09:43,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:09:43,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:09:44,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:09:45,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:50,128 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:09:51,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:52,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:09:56,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:09:58,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:09:58,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:09:58,352 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:10:02,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:02,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:10:07,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:12,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:13,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:10:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:10:13,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:10:13,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:10:18,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:20,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:10:23,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:10:24,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:29,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:29,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:10:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:10:29,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:10:33,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:10:34,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:39,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:43,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:10:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:10:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:10:44,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:10:50,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:10:50,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:51,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:10:55,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:10:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:10:59,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:10:59,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:11:00,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:03,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:11:05,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:11,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:13,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:11:14,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:11:14,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:11:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:11:17,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:20,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:11:21,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:11:23,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:28,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:29,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:11:29,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:11:29,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:11:30,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:11:33,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:38,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:40,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:11:43,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:44,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:11:44,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:11:44,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:11:49,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:50,145 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:11:50,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:11:55,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:11:59,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:11:59,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:11:59,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:12:00,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:04,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:12:05,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:10,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:13,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:12:13,216 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:12:13,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:12:13,219 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:12:13,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:12:14,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:12:14,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:12:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:12:14,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:12:16,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:20,146 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:12:21,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:24,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:12:26,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:12:29,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:12:29,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:12:32,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:37,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:12:37,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:42,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:12:44,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:12:44,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:12:47,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:12:48,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:50,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:12:54,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:12:57,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:12:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:12:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:12:59,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:13:00,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:05,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:07,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:13:10,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:13:14,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:13:14,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:13:16,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:17,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:13:20,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:13:21,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:26,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:27,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:13:29,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:13:29,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:13:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:13:32,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:35,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:13:37,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:42,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:13:44,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:13:44,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:13:48,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:50,148 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:13:54,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:59,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:13:59,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:13:59,915 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:13:59,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:14:05,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:10,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:14,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:14:14,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:14:14,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:14:16,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:19,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:14:20,148 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:14:21,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:26,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:29,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:14:29,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:14:29,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:14:29,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:14:32,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:37,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:38,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:14:42,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:44,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:14:44,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:14:44,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:14:48,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:48,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:14:50,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:14:53,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:58,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:14:59,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:14:59,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:14:59,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:14:59,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:15:04,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:08,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:15:09,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:14,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:15,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:15:15,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:15:15,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:15:19,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:15:19,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:20,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:15:25,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:27,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:15:30,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:15:30,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:15:30,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:15:30,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:36,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:37,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:15:41,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:45,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:15:45,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:15:45,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:15:46,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:47,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:15:50,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:15:52,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:15:57,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:15:58,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:00,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:16:00,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:16:00,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:16:03,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:04,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:16:04,596 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:16:04,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:16:04,598 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:16:05,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:16:05,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:07,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:09,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:14,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:15,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:15,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:16:15,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:16:15,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:16:20,150 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:16:20,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:25,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:25,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:30,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:30,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:16:30,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:16:30,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:16:35,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:36,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:41,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:16:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:16:45,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:16:46,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:47,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:50,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:16:52,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:16:55,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:16:57,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:00,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:17:00,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:17:00,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:17:02,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:05,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:17:08,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:13,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:15,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:17:15,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:17:15,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:17:15,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:17:18,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:20,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:17:23,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:25,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:17:28,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:30,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:17:30,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:17:30,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:17:34,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:35,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:17:39,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:44,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:45,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:17:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:17:45,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:17:45,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:17:50,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:17:50,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:17:53,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:17:55,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:00,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:00,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:18:00,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:18:00,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:18:03,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:18:05,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:10,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:13,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:18:15,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:18:15,685 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:18:15,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:18:15,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:20,154 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:18:21,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:23,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:18:26,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:30,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:18:30,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:18:30,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:18:31,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:33,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:18:36,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:41,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:43,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:18:45,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:18:45,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:18:45,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:18:47,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:50,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:18:52,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:18:53,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:18:57,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:01,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:19:01,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:19:01,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:19:02,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:03,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:13,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:13,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:16,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:19:16,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:19:16,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:19:18,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:20,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:19:21,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:19:21,997 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:19:21,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:19:21,999 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:19:22,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:19:22,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:24,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:24,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:29,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:31,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:19:31,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:19:31,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:19:34,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:34,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:39,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:42,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:46,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:19:46,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:19:46,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:19:50,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:19:50,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:19:51,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:19:55,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:00,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:01,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:20:01,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:20:01,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:20:01,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:20:05,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:10,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:11,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:20:15,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:16,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:20:16,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:20:16,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:20:20,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:20:21,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:21,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:20:26,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:29,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:20:31,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:20:31,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:31,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:20:31,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:20:36,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:39,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:20:42,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:20:46,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:20:46,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:20:47,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:49,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:20:50,158 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:20:53,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:59,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:20:59,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:21:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:21:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:21:01,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:21:04,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:09,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:21:10,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:15,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:16,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:21:16,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:21:16,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:21:17,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:21:20,158 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:21:21,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:26,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:27,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:21:31,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:31,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:21:31,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:21:31,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:21:36,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:37,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:21:41,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:46,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:21:46,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:21:46,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:21:48,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:21:50,159 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:21:52,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:57,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:21:58,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:01,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:22:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:22:01,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:22:02,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:08,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:08,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:13,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:22:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:22:16,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:22:18,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:18,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:20,159 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:22:24,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:28,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:29,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:31,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:22:31,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:22:31,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:22:34,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:36,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:22:36,016 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:22:36,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:22:36,019 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:22:36,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:22:37,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:38,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:40,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:45,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:46,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:22:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:22:46,709 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:22:47,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:22:50,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:22:50,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:55,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:22:57,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:23:01,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:01,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:23:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:23:01,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:23:06,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:07,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:23:11,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:15,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:23:16,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:23:16,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:16,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:23:16,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:23:20,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:23:22,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:25,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:23:27,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:31,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:23:31,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:23:31,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:23:33,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:35,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:23:39,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:44,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:45,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:23:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:23:46,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:23:46,710 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:23:49,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:50,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:23:55,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:23:55,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:00,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:02,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:24:02,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:24:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:24:05,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:05,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:10,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:14,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:16,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:17,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:24:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:24:17,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:24:20,163 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:24:22,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:24,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:27,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:32,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:24:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:24:32,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:24:32,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:34,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:38,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:43,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:44,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:47,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:24:47,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:24:47,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:24:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:24:50,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:24:54,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:24:55,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:00,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:02,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:25:02,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:25:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:25:04,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:25:05,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:10,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:14,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:25:16,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:25:17,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:25:17,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:25:20,166 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:25:22,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:24,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:25:27,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:32,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:25:32,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:25:32,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:25:33,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:25:37,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:41,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:25:43,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:47,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:25:47,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:25:47,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:25:48,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:50,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:25:50,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:25:50,957 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:25:50,958 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:25:50,960 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:25:51,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:25:51,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:25:53,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:25:58,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:01,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:26:02,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:26:02,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:26:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:26:04,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:09,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:11,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:26:14,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:26:17,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:26:17,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:26:20,167 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:26:20,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:25,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:30,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:32,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:26:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:26:32,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:26:32,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:26:36,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:41,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:42,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:26:46,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:48,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:26:48,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:26:48,820 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:26:50,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:26:52,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:26:52,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:26:57,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:00,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:27:02,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:27:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:27:03,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:27:08,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:12,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:27:13,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:18,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:27:18,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:27:18,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:27:18,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:20,169 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:27:24,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:26,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:27:29,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:27:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:27:33,817 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:27:34,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:27:34,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:39,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:44,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:27:45,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:48,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:27:48,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:27:48,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:27:50,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:27:50,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:27:54,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:27:55,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:00,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:28:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:28:03,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:28:04,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:28:06,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:11,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:14,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:28:16,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:18,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:28:18,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:28:18,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:28:20,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:28:21,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:23,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:28:26,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:31,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:28:32,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:33,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:28:33,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:28:33,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:28:38,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:41,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:28:43,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:48,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:49,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:28:49,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:28:49,254 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:28:50,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:28:51,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:28:54,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:28:59,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:02,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:29:04,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:29:04,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:29:04,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:29:05,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:10,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:12,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:29:15,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:19,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:29:19,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:29:19,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:29:20,173 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:29:20,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:29:20,346 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:29:20,346 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:29:20,348 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:29:21,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:29:21,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:22,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:29:26,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:31,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:32,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:29:34,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:29:34,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:29:34,254 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:29:37,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:42,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:29:42,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:47,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:49,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:29:49,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:29:49,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:29:50,173 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:29:50,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:29:53,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:29:58,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:00,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:30:03,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:04,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:30:04,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:30:04,256 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:30:08,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:10,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:30:13,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:19,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:19,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:30:19,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:30:19,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:30:20,174 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:30:20,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:30:24,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:29,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:30,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:30:34,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:30:34,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:30:34,259 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:30:35,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:38,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:30:40,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:45,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:48,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:30:49,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:30:49,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:30:49,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:30:50,174 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:30:51,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:30:56,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:01,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:04,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:31:04,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:31:04,263 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:31:06,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:10,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:31:11,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:16,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:19,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:31:19,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:31:19,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:31:20,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:31:20,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:31:22,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:27,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:30,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:31:32,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:34,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:31:34,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:31:34,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:31:38,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:40,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:31:43,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:48,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:31:49,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:31:49,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:31:49,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:31:49,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:31:50,188 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:31:55,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:00,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:00,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:04,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:32:04,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:32:04,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:32:05,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:10,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:10,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:15,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:19,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:32:19,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:32:19,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:32:20,188 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:32:20,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:21,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:26,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:28,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:31,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:32:35,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:32:35,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:32:36,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:38,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:41,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:46,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:46,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:32:46,789 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:32:46,789 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:32:46,792 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:32:47,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:32:47,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:48,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:50,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:32:50,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:32:50,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:32:50,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:32:50,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:51,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:32:52,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:32:56,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:01,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:05,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:33:05,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:33:05,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:33:07,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:12,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:17,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:20,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:33:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:33:20,592 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:33:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:33:22,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:27,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:32,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:35,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:33:35,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:33:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:33:37,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:42,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:47,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:50,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:33:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:33:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:33:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:33:53,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:33:58,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:03,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:05,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:34:05,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:34:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:34:09,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:14,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:19,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:20,190 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:34:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:34:20,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:34:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:34:25,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:30,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:34:35,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:34:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:34:35,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:40,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:45,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:50,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:34:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:34:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:34:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:34:50,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:34:55,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:00,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:05,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:35:05,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:35:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:35:06,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:11,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:16,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:20,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:35:20,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:35:20,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:35:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:35:22,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:27,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:32,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:35:35,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:35:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:35:38,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:43,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:48,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:50,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:35:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:35:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:35:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:35:54,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:35:59,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:04,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:05,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:36:05,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:36:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:36:09,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:14,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:19,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:20,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:36:20,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:36:20,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:36:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:36:24,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:29,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:34,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:36:35,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:36:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:36:40,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:45,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:50,194 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:36:50,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:36:50,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:36:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:36:50,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:36:55,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:00,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:05,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:37:05,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:37:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:37:06,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:11,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:16,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:20,195 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:37:20,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:37:20,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:37:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:37:21,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:26,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:31,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:37:35,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:37:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:37:36,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:41,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:46,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:50,195 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:37:50,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:37:50,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:37:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:37:52,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:37:57,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:02,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:05,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:38:05,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:38:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:38:07,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:12,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:17,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:20,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:38:20,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:38:20,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:38:20,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:38:23,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:28,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:33,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:35,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:38:35,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:38:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:38:39,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:44,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:49,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:50,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:38:50,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:38:50,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:38:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:38:54,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:38:59,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:04,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:05,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:39:05,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:39:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:39:09,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:14,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:19,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:20,198 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:39:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:39:20,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:39:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:39:25,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:30,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:35,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:39:35,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:39:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:39:35,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:40,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:45,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:50,198 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:39:50,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:39:50,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:39:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:39:51,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:39:56,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:01,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:05,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:40:05,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:40:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:40:06,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:11,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:16,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:20,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:40:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:40:20,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:40:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:40:21,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:26,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:31,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:35,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:40:35,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:40:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:40:37,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:42,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:47,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:50,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:40:50,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:40:50,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:40:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:40:52,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:40:57,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:02,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:05,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:41:05,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:41:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:41:07,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:12,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:17,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:20,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:41:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:41:20,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:41:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:41:23,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:28,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:33,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:35,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:41:35,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:41:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:41:39,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:44,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:49,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:50,200 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:41:50,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:41:50,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:41:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:41:54,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:41:59,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:04,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:05,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:42:05,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:42:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:42:10,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:15,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:20,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:42:20,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:42:20,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:42:20,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:42:20,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:25,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:30,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:35,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:42:35,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:42:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:42:36,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:41,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:46,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:50,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:42:50,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:42:50,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:42:50,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:42:51,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:42:56,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:01,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:05,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:43:05,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:43:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:43:06,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:11,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:11,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:43:11,963 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:43:11,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:43:11,966 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:43:12,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:43:13,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:43:14,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:43:17,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:20,202 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:43:20,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:43:20,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:43:20,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:43:22,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:27,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:32,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:35,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:43:35,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:43:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:43:38,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:41,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:43:43,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:48,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:50,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:43:50,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:43:50,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:43:50,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:43:53,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:43:58,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:03,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:06,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:44:06,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:44:07,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:44:09,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:14,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:19,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:20,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:44:21,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:44:21,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:44:21,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:44:25,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:30,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:33,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:44:36,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:39,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:44:39,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:44:39,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:44:41,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:44:41,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:46,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:46,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:44:50,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:44:52,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:44:57,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:00,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:45:00,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:45:00,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:45:00,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:45:02,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:03,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:45:07,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:12,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:15,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:45:15,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:45:15,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:45:15,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:45:18,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:20,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:45:24,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:30,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:45:30,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:45:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:45:30,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:45:34,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:39,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:44,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:45,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:45:45,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:45:45,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:45:45,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:45:49,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:45:50,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:45:55,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:00,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:00,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:46:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:46:00,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:46:00,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:46:03,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:46:05,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:10,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:15,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:46:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:46:15,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:46:15,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:46:15,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:15,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:46:20,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:46:21,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:25,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:46:26,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:30,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:46:30,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:46:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:46:30,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:46:31,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:35,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:46:37,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:42,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:45,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:46:45,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:46:45,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:46:45,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:46:47,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:47,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:46:50,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:46:53,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:46:57,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:46:58,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:47:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:47:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:47:00,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:47:03,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:05,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:47:09,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:14,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:47:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:47:15,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:47:15,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:47:16,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:47:19,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:20,207 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:47:24,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:26,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:47:29,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:47:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:47:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:47:30,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:47:35,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:36,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:47:40,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:45,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:45,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:47:45,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:47:45,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:47:45,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:47:48,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:47:50,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:47:50,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:55,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:47:57,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:48:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:48:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:48:00,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:48:00,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:48:00,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:05,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:07,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:48:11,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:15,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:48:15,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:48:15,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:48:15,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:48:17,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:17,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:48:20,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:48:22,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:27,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:48:27,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:30,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:48:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:48:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:48:30,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:48:33,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:39,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:39,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:48:44,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:46,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:48:46,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:48:46,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:48:46,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:48:49,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:48:49,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:50,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:48:55,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:48:59,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:00,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:01,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:49:01,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:49:01,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:49:06,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:09,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:12,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:17,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:49:17,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:49:17,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:49:17,298 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:49:17,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:19,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:20,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:49:23,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:28,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:49:28,638 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:49:28,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:49:28,641 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:49:28,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:49:29,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:30,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:32,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:49:32,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:49:32,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:49:33,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:38,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:42,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:43,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:48,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:49:48,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:49:48,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:49:48,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:49:49,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:49:50,210 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:49:52,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:49:55,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:00,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:02,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:50:03,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:50:03,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:50:03,726 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:50:03,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:50:06,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:12,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:50:12,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:17,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:19,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:50:19,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:50:19,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:50:19,592 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:50:20,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:50:22,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:50:22,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:27,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:32,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:50:33,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:34,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:50:34,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:50:34,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:50:34,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:50:38,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:42,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:50:44,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:49,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:50,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:50:50,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:50:50,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:50:50,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:50:50,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:50:51,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:50:54,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:50:59,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:03,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:51:04,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:05,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:51:05,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:51:05,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:51:05,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:51:10,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:13,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:51:16,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:20,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:51:20,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:51:20,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:51:20,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:51:20,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:51:21,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:23,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:51:27,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:32,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:33,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:51:35,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:51:35,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:51:35,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:51:35,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:51:37,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:42,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:43,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:51:48,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:50,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:51:50,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:51:50,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:51:50,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:51:50,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:51:53,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:51:54,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:51:58,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:03,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:05,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:52:05,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:52:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:52:06,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:08,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:13,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:16,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:18,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:20,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:52:20,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:52:20,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:52:20,424 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:52:24,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:26,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:29,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:34,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:52:35,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:52:35,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:52:36,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:40,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:46,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:46,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:50,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:52:50,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:52:50,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:52:50,439 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:52:51,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:52:54,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:52:54,127 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:52:54,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:52:54,129 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:52:54,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:52:55,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:56,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:52:57,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:02,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:53:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:53:05,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:53:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:53:07,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:08,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:53:12,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:17,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:18,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:53:20,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:53:20,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:53:20,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:53:20,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:53:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:27,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:53:29,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:34,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:53:35,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:53:35,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:53:39,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:44,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:49,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:50,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:53:50,216 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:53:50,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:53:50,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:53:50,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:53:54,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:53:59,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:04,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:54:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:54:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:54:05,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:54:05,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:54:05,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:10,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:14,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:54:16,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:20,218 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:54:21,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:54:21,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:54:21,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:54:21,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:54:21,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:24,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:54:26,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:31,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:36,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:54:36,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:54:36,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:54:36,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:54:36,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:36,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:54:41,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:44,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:54:47,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:50,219 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:54:51,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:54:51,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:54:51,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:54:51,787 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:54:52,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:54:54,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:54:58,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:03,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:04,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:55:06,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:55:06,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:55:06,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:55:06,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:55:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:14,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:55:14,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:19,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:20,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:55:21,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:55:21,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:55:21,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:55:21,789 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:55:23,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:55:24,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:29,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:33,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:55:34,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:36,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:55:36,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:55:36,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:55:36,786 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:55:40,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:45,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:46,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:55:50,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:55:51,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:55:52,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:55:52,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:55:52,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:55:52,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:55:56,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:55:56,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:01,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:07,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:56:07,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:56:07,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:56:07,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:56:11,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:12,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:56:16,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:20,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:56:22,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:22,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:56:22,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:56:22,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:56:22,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:56:27,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:32,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:56:33,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:38,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:56:38,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:56:38,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:56:38,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:56:39,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:40,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:56:44,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:49,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:50,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 18:56:50,011 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 18:56:50,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 18:56:50,014 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 18:56:50,222 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:56:50,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 18:56:51,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:56:52,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:56:53,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:56:53,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:56:53,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:56:53,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:56:54,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:56:59,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:02,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:04,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:08,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:57:08,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:57:08,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:57:08,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:57:10,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:12,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:15,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:20,223 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:57:20,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:22,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:23,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:57:23,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:57:23,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:57:23,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:57:26,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:26,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:31,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:34,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:36,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:38,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:57:38,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:57:38,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:57:42,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:45,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:47,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:50,225 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:57:53,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:57:53,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:57:53,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:57:53,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:57:54,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:57:58,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:04,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:04,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:58:08,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:58:08,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:58:08,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:58:09,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:14,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:58:14,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:19,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:20,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:58:22,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:58:23,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:58:23,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:58:23,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:58:25,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:30,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:32,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:58:36,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:38,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:58:38,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:58:38,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:58:41,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:42,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:58:46,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:50,228 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:58:52,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:58:52,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:58:53,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:58:53,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:58:53,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:58:57,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:02,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:59:03,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:08,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:09,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:59:09,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:59:09,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:59:09,826 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:59:12,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:59:13,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:18,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:20,228 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:59:22,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:59:24,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:24,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:59:24,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:59:24,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:59:24,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:59:30,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:32,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:59:35,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:40,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:59:40,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:59:40,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:59:40,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:59:41,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:42,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:59:46,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:50,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 18:59:52,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 18:59:53,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 18:59:55,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 18:59:55,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 18:59:55,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 18:59:55,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 18:59:57,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:02,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:03,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:07,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:10,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:00:10,011 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:00:10,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:00:10,015 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:00:10,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:00:10,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:00:10,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:00:10,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:00:12,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:13,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:18,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:20,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:20,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:00:24,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:25,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:00:25,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:00:25,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:00:29,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:32,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:35,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:40,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:40,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:00:40,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:00:40,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:00:42,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:45,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:50,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:00:50,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:00:52,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:00:55,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:00:55,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:00:55,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:00:56,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:02,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:02,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:01:07,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:10,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:01:10,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:01:10,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:01:10,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:01:12,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:12,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:01:17,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:20,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:01:22,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:01:23,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:25,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:01:25,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:01:25,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:01:25,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:01:28,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:32,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:01:33,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:38,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:40,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:01:40,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:01:40,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:01:40,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:01:42,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:01:44,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:49,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:01:50,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:01:51,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:01:55,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:01:55,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:01:55,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:01:55,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:01:55,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:01,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:01,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:06,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:10,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:02:10,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:02:10,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:02:10,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:02:11,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:11,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:16,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:20,246 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:02:21,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:22,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:25,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:02:25,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:02:25,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:02:25,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:02:27,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:31,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:33,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:38,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:40,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:02:40,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:02:40,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:02:40,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:02:40,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:43,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:48,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:48,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:50,236 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:02:54,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:02:56,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:02:56,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:02:56,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:02:56,481 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:02:58,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:02:59,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:04,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:08,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:03:10,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:11,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:03:11,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:03:11,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:03:11,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:03:15,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:18,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:03:20,236 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:03:21,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:26,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:26,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:03:26,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:03:26,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:03:26,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:03:27,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:03:27,687 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:03:27,687 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:03:27,690 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:03:27,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:03:28,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:03:29,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:03:31,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:36,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:38,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:03:41,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:03:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:03:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:03:41,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:03:42,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:47,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:48,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:03:50,238 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:03:53,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:56,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:03:56,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:03:56,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:03:56,586 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:03:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:03:59,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:03,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:07,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:08,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:11,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:04:11,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:04:11,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:04:11,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:04:14,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:19,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:20,238 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:04:20,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:25,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:26,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:04:26,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:04:26,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:04:27,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:30,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:36,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:37,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:41,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:04:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:04:41,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:04:46,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:47,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:50,238 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:04:52,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:04:56,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:04:56,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:04:56,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:04:56,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:04:57,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:04:57,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:02,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:07,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:05:08,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:11,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:05:11,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:05:11,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:05:11,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:05:13,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:16,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:05:19,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:20,239 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:05:25,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:26,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:05:26,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:05:26,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:05:26,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:05:30,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:34,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:05:36,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:41,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:42,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:05:42,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:05:42,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:05:42,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:05:44,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:05:46,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:50,239 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:05:52,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:54,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:05:57,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:05:57,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:05:57,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:05:57,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:06:02,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:06,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:08,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:13,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:06:13,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:06:13,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:06:13,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:06:14,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:16,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:19,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:20,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:06:25,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:25,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:28,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:06:28,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:06:28,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:06:28,230 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:06:30,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:36,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:37,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:41,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:43,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:06:43,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:06:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:06:43,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:06:44,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:06:44,216 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:06:44,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:06:44,218 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:06:45,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:06:45,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:46,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:47,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:50,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:06:52,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:57,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:06:57,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:06:58,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:06:58,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:06:58,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:06:58,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:07:03,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:07,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:07:08,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:13,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:07:13,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:07:13,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:07:13,234 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:07:14,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:15,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:07:19,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:20,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:07:25,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:25,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:07:28,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:07:28,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:07:28,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:07:28,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:07:30,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:35,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:07:35,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:40,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:43,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:07:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:07:43,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:07:43,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:07:45,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:07:45,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:50,242 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:07:51,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:55,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:07:56,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:07:58,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:07:58,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:07:58,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:07:58,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:08:01,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:05,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:08:06,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:11,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:13,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:08:13,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:08:13,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:08:13,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:08:14,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:08:17,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:20,245 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:08:23,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:24,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:08:28,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:28,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:08:28,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:08:28,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:08:28,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:08:33,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:34,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:08:38,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:43,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:08:43,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:08:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:08:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:08:43,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:08:44,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:49,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:50,246 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:08:54,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:08:55,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:08:58,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:08:58,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:08:58,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:08:58,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:09:00,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:03,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:09:06,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:11,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:13,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:09:13,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:09:13,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:09:13,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:09:13,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:09:16,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:20,248 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:09:21,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:23,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:09:26,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:28,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:09:28,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:09:28,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:09:28,236 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:09:32,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:33,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:09:37,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:42,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:09:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:09:43,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:09:43,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:09:43,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:09:48,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:50,248 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:09:53,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:09:54,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:09:58,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:09:58,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:09:58,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:09:58,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:09:59,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:01,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:10:01,336 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:10:01,337 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:10:01,339 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:10:01,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:10:02,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:03,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:05,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:10,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:13,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:10:13,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:10:13,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:10:13,234 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:10:13,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:15,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:20,249 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:10:20,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:23,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:25,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:29,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:10:29,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:10:29,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:10:29,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:10:31,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:31,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:36,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:41,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:44,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:10:44,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:10:44,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:10:44,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:10:45,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:46,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:50,259 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:10:52,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:10:57,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:10:59,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:10:59,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:10:59,062 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:11:03,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:07,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:11:08,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:13,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:14,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:11:14,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:11:14,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:11:14,064 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:11:15,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:11:19,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:20,249 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:11:24,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:25,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:11:29,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:11:29,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:11:29,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:11:30,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:35,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:11:35,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:40,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:44,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:11:44,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:11:44,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:11:45,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:11:46,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:50,251 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:11:52,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:55,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:11:58,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:11:59,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:11:59,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:11:59,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:12:03,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:12:03,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:08,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:13,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:12:14,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:14,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:12:14,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:12:14,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:12:19,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:20,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:12:23,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:12:24,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:29,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:30,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:12:30,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:12:30,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:12:30,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:12:33,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:12:35,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:40,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:43,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:12:45,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:12:45,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:12:45,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:12:45,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:12:46,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:50,254 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:12:51,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:12:53,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:12:56,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:00,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:13:00,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:13:00,304 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:13:02,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:03,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:07,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:11,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:12,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:15,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:13:15,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:13:15,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:13:18,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:20,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:13:20,106 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:13:20,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:13:20,108 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:13:20,254 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:13:20,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:13:21,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:22,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:24,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:29,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:30,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:13:30,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:13:30,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:13:32,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:34,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:39,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:42,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:45,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:13:45,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:13:45,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:13:45,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:50,255 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:13:51,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:13:52,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:13:56,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:00,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:14:00,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:14:00,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:14:01,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:14:01,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:06,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:11,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:14:12,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:15,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:14:15,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:14:15,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:14:17,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:19,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:14:20,255 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:14:23,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:28,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:29,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:14:30,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:14:30,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:14:30,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:14:33,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:38,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:39,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:14:43,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:45,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:14:45,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:14:45,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:14:49,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:49,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:14:50,256 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:14:55,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:14:59,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:15:00,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:15:00,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:15:00,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:15:00,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:05,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:09,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:15:11,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:15,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:15:15,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:15:15,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:15:15,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:15:16,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:17,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:15:20,257 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:15:22,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:27,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:27,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:15:30,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:15:30,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:15:30,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:15:32,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:37,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:38,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:15:42,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:45,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:15:45,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:15:45,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:15:45,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:15:48,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:15:48,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:50,257 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:15:54,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:15:58,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:00,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:00,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:16:00,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:16:00,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:16:05,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:08,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:10,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:15,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:16:15,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:16:15,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:16:15,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:16,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:20,257 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:16:21,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:26,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:28,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:30,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:16:30,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:16:30,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:16:31,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:35,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:16:35,426 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:16:35,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:16:35,428 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:16:36,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:16:37,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:37,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:42,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:45,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:45,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:16:45,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:16:45,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:16:47,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:50,258 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:16:53,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:16:55,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:16:58,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:00,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:17:00,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:17:00,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:17:00,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:17:03,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:05,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:17:08,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:13,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:15,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:17:15,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:17:15,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:17:15,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:17:18,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:20,260 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:17:24,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:25,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:17:29,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:31,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:17:31,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:17:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:17:31,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:17:34,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:17:34,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:39,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:44,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:17:45,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:46,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:17:46,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:17:46,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:17:46,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:17:50,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:17:51,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:17:54,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:17:56,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:01,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:01,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:18:01,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:18:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:18:01,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:18:04,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:18:07,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:12,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:14,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:18:16,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:18:16,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:18:16,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:18:16,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:18:18,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:20,263 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:18:23,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:24,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:18:28,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:31,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:18:31,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:18:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:18:31,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:18:34,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:34,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:18:39,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:44,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:44,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:18:46,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:18:46,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:18:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:18:46,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:18:50,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:18:50,264 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:18:53,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:18:55,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:01,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:01,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:19:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:19:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:19:01,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:19:03,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:06,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:11,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:13,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:16,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:16,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:19:16,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:19:16,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:19:16,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:19:20,265 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:19:21,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:22,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:27,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:31,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:19:31,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:19:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:19:31,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:19:31,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:32,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:37,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:41,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:43,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:46,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:19:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:19:46,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:19:46,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:19:49,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:50,265 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:19:50,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:19:50,267 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:19:50,267 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:19:50,269 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:19:50,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:19:50,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:52,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:19:54,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:19:59,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:01,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:20:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:20:01,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:20:01,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:20:02,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:20:05,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:10,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:12,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:20:15,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:18,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:20:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:20:18,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:20:18,388 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:20:20,266 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:20:20,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:22,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:20:25,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:30,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:20:31,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:33,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:20:33,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:20:33,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:20:33,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:20:36,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:40,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:20:42,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:47,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:48,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:20:48,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:20:48,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:20:48,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:20:50,266 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:20:50,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:20:53,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:20:58,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:00,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:03,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:21:03,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:21:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:21:03,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:21:03,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:08,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:10,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:13,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:18,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:21:18,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:21:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:21:18,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:21:18,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:19,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:20,267 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:21:25,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:28,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:31,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:33,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:21:33,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:21:33,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:21:33,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:21:36,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:38,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:42,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:48,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:48,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:21:48,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:21:48,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:21:48,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:21:48,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:50,267 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:21:53,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:21:57,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:21:58,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:22:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:22:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:22:03,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:22:03,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:07,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:22:09,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:14,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:18,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:22:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:22:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:22:18,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:22:18,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:22:20,268 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:22:20,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:25,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:28,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:22:31,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:33,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:22:33,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:22:33,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:22:33,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:22:36,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:38,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:22:41,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:46,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:48,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:22:48,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:22:48,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:22:48,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:22:48,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:22:50,268 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:22:52,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:57,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:22:58,325 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:02,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:03,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:23:03,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:23:03,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:23:03,390 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:23:06,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:23:06,456 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:23:06,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:23:06,459 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:23:07,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:23:07,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:08,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:08,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:13,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:23:18,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:23:18,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:23:18,388 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:23:18,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:18,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:20,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:23:24,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:26,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:29,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:33,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:23:34,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:23:34,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:23:34,002 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:23:35,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:36,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:40,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:45,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:46,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:48,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:23:48,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:23:49,000 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:23:50,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:23:51,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:23:56,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:23:56,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:01,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:04,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:24:04,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:24:04,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:24:04,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:24:06,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:24:06,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:11,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:16,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:24:17,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:18,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:24:19,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:24:19,001 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:24:20,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:24:23,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:26,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:24:28,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:33,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:34,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:24:34,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:24:34,002 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:24:37,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:24:39,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:44,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:46,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:24:49,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:24:49,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:24:49,001 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:24:50,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:50,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:24:55,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:24:56,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:25:00,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:04,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:25:04,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:25:04,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:25:04,221 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:25:06,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:25:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:11,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:16,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:19,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:25:19,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:25:19,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:25:19,221 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:25:20,271 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:25:20,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:25:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:27,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:30,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:25:33,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:34,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:25:34,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:25:34,219 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:25:38,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:40,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:25:43,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:49,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:25:49,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:25:49,219 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:25:49,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:25:50,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:25:50,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:25:55,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:00,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:00,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:04,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:26:04,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:26:04,219 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:26:06,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:10,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:11,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:16,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:18,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:19,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:26:19,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:26:19,219 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:26:20,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:26:22,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:27,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:26:27,316 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:26:27,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:26:27,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:27,319 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:26:27,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:26:28,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:29,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:32,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:34,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:26:34,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:26:34,220 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:26:37,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:38,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:42,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:48,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:49,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:26:49,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:26:49,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:26:49,219 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:26:50,273 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:26:53,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:58,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:26:59,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:03,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:05,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:27:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:27:05,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:27:05,094 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:27:09,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:09,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:14,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:17,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:19,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:20,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:27:20,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:27:20,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:27:20,091 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:27:20,284 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:27:25,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:26,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:30,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:35,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:27:35,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:27:35,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:27:35,097 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:27:35,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:36,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:40,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:46,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:46,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:50,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:27:50,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:27:50,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:27:50,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:27:50,282 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:27:52,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:27:56,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:27:58,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:03,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:05,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:28:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:28:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:28:05,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:28:06,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:28:08,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:13,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:16,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:28:19,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:20,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:28:20,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:28:20,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:28:20,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:28:20,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:28:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:24,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:28:29,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:34,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:28:35,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:35,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:28:35,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:28:35,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:28:35,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:28:40,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:44,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:28:45,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:50,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:28:50,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:28:50,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:28:50,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:28:50,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:28:51,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:28:54,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:28:56,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:01,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:04,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:29:05,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:29:05,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:29:05,092 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:29:07,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:12,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:15,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:17,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:20,277 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:29:20,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:29:20,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:29:20,699 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:29:20,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:29:22,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:23,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:27,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:33,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:33,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:35,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:29:35,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:29:35,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:29:38,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:42,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:29:42,147 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:29:42,148 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:29:42,151 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:29:42,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:29:43,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:44,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:44,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:49,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:50,279 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:29:50,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:29:50,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:29:50,699 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:29:53,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:29:54,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:29:59,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:03,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:30:05,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:05,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:30:05,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:30:05,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:30:10,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:13,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:30:15,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:20,279 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:30:20,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:30:20,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:30:20,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:30:20,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:30:20,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:25,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:30,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:30:31,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:35,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:30:35,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:30:35,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:30:36,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:42,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:42,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:30:47,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:50,280 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:30:50,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:30:50,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:30:50,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:30:52,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:30:53,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:30:58,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:02,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:31:03,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:05,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:31:05,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:31:05,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:31:08,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:12,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:31:14,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:19,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:20,282 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:31:20,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:31:20,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:31:20,700 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:31:22,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:31:24,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:29,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:35,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:31:35,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:35,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:31:35,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:31:35,699 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:31:40,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:45,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:31:45,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:50,283 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:31:51,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:31:51,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:31:51,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:31:51,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:31:51,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:31:55,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:31:56,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:01,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:05,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:32:06,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:32:06,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:32:06,405 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:32:07,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:12,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:15,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:32:17,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:20,285 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:32:21,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:32:21,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:32:21,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:32:21,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:32:23,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:32:23,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:28,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:33,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:35,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:32:36,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:32:36,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:32:36,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:32:39,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:43,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:32:45,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:50,286 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:32:51,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:32:51,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:32:51,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:32:51,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:32:53,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:32:56,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:01,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:33:01,671 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:33:01,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:33:01,674 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:33:02,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:33:02,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:03,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:33:06,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:33:06,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:33:06,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:33:08,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:13,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:33:14,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:19,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:20,286 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:33:20,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:33:21,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:33:21,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:33:21,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:33:24,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:29,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:32,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:33:35,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:36,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:33:36,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:33:36,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:33:40,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:42,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:33:45,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:50,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:33:50,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:33:51,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:33:51,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:33:51,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:33:51,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:33:56,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:01,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:34:01,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:06,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:34:06,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:34:06,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:34:06,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:34:07,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:11,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:34:13,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:18,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:20,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:34:21,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:34:21,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:34:21,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:34:21,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:34:23,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:28,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:31,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:34:33,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:36,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:34:36,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:34:36,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:34:36,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:34:39,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:41,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:34:44,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:49,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:34:50,288 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:34:51,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:34:51,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:34:51,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:34:51,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:34:55,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:00,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:01,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:35:05,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:07,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:35:07,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:35:07,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:35:07,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:35:09,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:35:11,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:16,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:19,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:35:20,288 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:35:21,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:22,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:35:22,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:35:22,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:35:26,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:29,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:35:32,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:37,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:35:37,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:35:37,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:35:37,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:47,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:50,301 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:35:52,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:35:52,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:35:52,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:35:52,158 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:35:52,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:35:53,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:35:57,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:02,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:03,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:07,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:36:07,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:36:07,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:36:07,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:36:07,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:12,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:13,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:18,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:20,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:36:22,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:36:22,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:36:22,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:36:22,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:36:23,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:23,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:28,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:32,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:36:32,656 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:36:32,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:36:32,659 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:36:33,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:36:33,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:34,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:34,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:37,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:36:37,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:36:37,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:36:37,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:36:40,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:43,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:46,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:50,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:36:51,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:36:52,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:36:52,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:36:52,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:36:52,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:36:53,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:36:56,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:01,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:03,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:37:06,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:07,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:37:07,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:37:07,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:37:07,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:37:12,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:13,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:37:17,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:20,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:37:22,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:37:22,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:37:22,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:37:22,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:37:22,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:23,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:37:27,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:33,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:33,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:37:37,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:37:37,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:37:37,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:37:37,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:37:38,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:41,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:37:43,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:50,293 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:37:51,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:37:52,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:37:52,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:37:52,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:37:52,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:37:54,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:37:59,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:01,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:38:04,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:08,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:38:08,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:38:08,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:38:08,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:38:10,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:11,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:38:15,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:20,293 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:38:21,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:21,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:38:23,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:38:23,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:38:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:38:23,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:38:26,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:31,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:38:32,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:37,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:38:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:38:38,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:38:38,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:38:41,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:38:43,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:48,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:50,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:38:52,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:38:53,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:38:53,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:38:53,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:38:53,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:38:53,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:38:58,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:00,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:03,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:08,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:39:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:39:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:39:08,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:39:09,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:11,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:14,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:19,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:19,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:20,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:39:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:39:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:39:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:39:23,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:39:25,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:29,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:31,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:36,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:38,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:39:38,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:39:38,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:39:38,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:39:39,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:41,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:46,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:48,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:39:48,067 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:39:48,067 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:39:48,070 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:39:48,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:39:49,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:50,295 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:39:50,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:39:52,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:39:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:39:53,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:39:53,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:39:53,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:39:57,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:03,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:03,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:40:08,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:40:08,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:40:08,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:40:08,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:40:08,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:13,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:40:13,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:18,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:20,296 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:40:23,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:40:23,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:40:23,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:40:23,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:40:23,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:40:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:29,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:33,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:40:34,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:38,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:40:38,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:40:38,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:40:38,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:40:40,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:43,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:40:46,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:50,297 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:40:51,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:40:52,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:40:53,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:40:53,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:40:53,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:40:53,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:40:57,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:02,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:03,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:41:07,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:08,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:41:08,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:41:08,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:41:08,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:41:13,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:14,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:41:18,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:20,298 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:41:22,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:41:23,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:41:23,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:41:23,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:41:23,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:41:23,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:28,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:32,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:41:33,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:38,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:41:38,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:41:38,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:41:38,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:41:39,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:44,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:41:45,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:50,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:41:51,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:41:52,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:41:53,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:41:53,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:41:53,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:41:53,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:41:56,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:01,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:03,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:42:07,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:08,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:42:08,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:42:08,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:42:08,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:42:12,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:13,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:42:17,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:20,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:42:21,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:42:23,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:23,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:42:23,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:42:23,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:42:23,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:42:28,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:31,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:42:34,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:39,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:39,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:42:39,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:42:39,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:42:39,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:42:41,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:42:44,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:49,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:42:50,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:42:51,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:42:54,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:42:54,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:42:54,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:42:54,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:42:54,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:00,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:01,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:05,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:09,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:43:09,065 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:43:09,066 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:43:09,068 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:43:09,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:43:09,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:43:09,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:43:09,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:43:09,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:43:09,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:10,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:10,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:15,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:19,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:20,301 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:43:21,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:24,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:43:24,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:43:24,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:43:24,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:43:26,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:30,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:32,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:37,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:39,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:43:39,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:43:39,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:43:39,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:43:40,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:42,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:47,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:50,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:50,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:43:53,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:43:54,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:43:54,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:43:54,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:43:54,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:43:58,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:43:59,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:04,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:08,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:44:09,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:44:09,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:44:09,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:44:09,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:14,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:18,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:44:20,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:20,302 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:44:25,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:44:25,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:44:25,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:44:25,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:44:25,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:28,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:44:30,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:35,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:37,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:44:40,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:44:40,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:44:40,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:44:41,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:46,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:47,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:44:50,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:44:52,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:44:55,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:44:55,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:44:55,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:44:55,241 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:44:55,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:44:57,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:02,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:05,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:45:07,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:10,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:45:10,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:45:10,239 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:45:13,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:15,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:45:19,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:20,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:45:24,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:25,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:45:25,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:45:25,239 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:45:25,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:45:29,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:34,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:35,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:45:39,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:40,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:45:40,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:45:40,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:45:44,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:45:45,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:50,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:45:50,305 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:45:54,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:45:55,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:45:55,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:45:55,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:45:55,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:00,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:04,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:06,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:11,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:46:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:46:11,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:46:11,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:46:11,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:14,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:17,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:20,306 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:46:21,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:46:21,955 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:46:21,956 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:46:21,959 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:46:22,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:46:22,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:23,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:24,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:26,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:46:26,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:46:26,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:46:26,014 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:46:28,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:33,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:34,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:38,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:41,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:46:41,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:46:41,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:46:41,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:46:44,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:44,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:49,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:50,308 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:46:54,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:46:54,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:46:56,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:46:56,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:46:56,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:46:56,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:47:00,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:04,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:47:05,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:10,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:11,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:47:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:47:11,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:47:11,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:47:12,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:47:15,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:20,309 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:47:21,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:22,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:47:26,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:47:26,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:47:26,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:47:26,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:47:27,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:31,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:47:32,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:41,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:47:41,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:47:41,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:47:41,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:47:43,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:48,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:50,309 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:47:51,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:47:53,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:47:56,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:47:56,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:47:56,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:47:56,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:47:59,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:01,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:48:04,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:09,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:10,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:48:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:48:11,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:48:11,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:48:15,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:20,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:48:20,309 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:48:20,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:25,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:26,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:48:26,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:48:26,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:48:31,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:32,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:48:36,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:41,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:48:41,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:48:41,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:48:41,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:42,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:48:46,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:50,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:48:50,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:48:51,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:48:56,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:48:56,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:48:56,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:48:57,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:00,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:49:02,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:07,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:10,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:49:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:49:11,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:49:11,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:49:13,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:18,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:20,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:49:20,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:49:24,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:26,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:49:26,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:49:26,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:49:29,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:30,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:49:34,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:39,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:49:39,033 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:49:39,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:49:39,036 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:49:39,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:49:40,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:40,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:49:41,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:49:41,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:49:41,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:49:45,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:50,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:49:50,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:50,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:49:55,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:49:57,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:49:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:49:57,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:49:57,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:50:00,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:50:01,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:06,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:08,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:50:11,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:12,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:50:12,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:50:12,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:50:17,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:17,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:50:20,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:50:22,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:27,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:50:27,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:50:27,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:50:27,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:50:27,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:28,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:50:32,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:38,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:50:38,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:42,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:50:42,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:50:42,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:50:44,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:48,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:50:50,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:50,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:50:55,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:50:57,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:50:57,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:50:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:50:58,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:51:00,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:05,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:08,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:51:11,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:12,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:51:12,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:51:12,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:51:16,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:18,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:51:20,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:51:21,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:26,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:27,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:51:27,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:51:27,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:51:28,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:51:32,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:37,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:38,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:51:42,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:51:42,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:51:42,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:51:43,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:48,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:51:49,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:50,313 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:51:54,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:51:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:51:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:51:57,105 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:51:58,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:00,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:05,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:08,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:10,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:12,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:52:12,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:52:12,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:52:16,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:16,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:20,324 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:52:22,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:26,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:27,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:52:27,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:52:27,106 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:52:27,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:52:28,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:33,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:37,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:39,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:43,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:52:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:52:43,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:52:43,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:52:44,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:47,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:49,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:50,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:52:54,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:52:54,160 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:52:54,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:52:54,163 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:52:55,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:52:55,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:55,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:52:56,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:52:58,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:52:58,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:52:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:52:58,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:53:00,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:06,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:53:06,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:11,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:13,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:53:13,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:53:13,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:53:13,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:53:16,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:53:17,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:20,315 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:53:23,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:26,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:53:28,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:53:28,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:53:28,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:53:28,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:53:28,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:33,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:36,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:53:38,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:53:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:53:43,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:53:43,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:53:43,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:46,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:53:48,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:50,315 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:53:54,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:53:56,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:53:58,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:53:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:53:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:53:58,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:54:00,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:04,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:54:06,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:11,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:13,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:54:13,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:54:13,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:54:13,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:54:14,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:54:17,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:20,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:54:23,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:28,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:54:28,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:54:28,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:54:28,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:54:28,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:28,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:54:33,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:39,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:41,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:54:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:54:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:54:43,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:54:43,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:54:44,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:50,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:54:50,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:51,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:54:55,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:54:58,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:54:58,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:54:58,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:54:58,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:55:00,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:01,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:55:05,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:10,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:11,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:55:13,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:55:13,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:55:13,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:55:13,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:55:15,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:20,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:55:20,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:55:21,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:26,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:28,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:55:28,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:55:28,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:55:28,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:55:30,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:55:31,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:36,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:38,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:55:42,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:43,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:55:43,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:55:43,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:55:43,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:55:47,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:50,317 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:55:50,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:55:53,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:58,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:55:58,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:55:58,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:55:58,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:55:58,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:56:00,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:03,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:08,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:08,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:13,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:56:13,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:56:13,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:56:13,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:56:14,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:17,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:56:17,312 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:56:17,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:56:17,316 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:56:17,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:56:18,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:19,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:20,317 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:56:20,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:25,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:28,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:56:28,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:56:28,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:56:28,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:56:28,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:30,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:35,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:38,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:41,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:43,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:56:43,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:56:43,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:56:43,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:56:46,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:48,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:56:50,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:56:52,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:57,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:56:58,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:56:58,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:56:58,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:56:58,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:56:59,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:02,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:07,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:07,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:12,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:14,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:57:14,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:57:14,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:57:14,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:57:17,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:18,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:20,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:57:24,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:27,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:29,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:57:29,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:57:29,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:57:29,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:57:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:34,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:37,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:39,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:44,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:57:44,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:57:44,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:57:44,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:57:45,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:47,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:50,319 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:57:51,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:56,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:57:57,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:57:59,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:57:59,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:57:59,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:57:59,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:58:02,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:07,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:58:08,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:13,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:14,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:58:14,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:58:14,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:58:14,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:58:16,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:58:18,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:20,319 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:58:23,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:24,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:58:28,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:29,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:58:29,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:58:29,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:58:29,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:58:34,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:34,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:58:39,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:44,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:58:44,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:58:44,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:58:44,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:58:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:44,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:58:49,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:50,320 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:58:54,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:58:55,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:58:59,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:58:59,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:58:59,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:58:59,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:59:00,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:04,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:06,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:11,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:14,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:59:14,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:59:14,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:59:14,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:59:15,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:16,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:20,323 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:59:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:25,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:27,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:29,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:59:29,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:59:29,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:59:29,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:59:32,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 19:59:32,256 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 19:59:32,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 19:59:32,259 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 19:59:33,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 19:59:33,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:34,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:35,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:38,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:43,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:44,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:59:44,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:59:44,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:59:44,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:59:45,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:48,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:50,324 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 19:59:54,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 19:59:55,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 19:59:59,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 19:59:59,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 19:59:59,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 19:59:59,417 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 19:59:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:05,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:05,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:00:10,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:13,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:00:14,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:00:14,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:00:14,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:00:14,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:00:15,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:20,325 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:00:21,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:23,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:00:26,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:29,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:00:29,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:00:29,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:00:29,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:00:32,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:35,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:00:38,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:43,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:44,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:00:44,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:00:44,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:00:44,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:00:44,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:00:49,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:00:50,326 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:00:54,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:00:55,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:00,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:00,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:01:00,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:01:00,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:01:00,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:01:02,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:01:06,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:11,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:12,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:01:15,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:01:15,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:01:15,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:01:15,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:01:16,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:20,327 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:01:21,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:23,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:01:27,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:30,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:01:30,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:01:30,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:01:30,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:01:32,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:33,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:01:37,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:43,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:01:43,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:45,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:01:45,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:01:45,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:01:45,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:01:49,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:50,328 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:01:53,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:01:54,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:01:59,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:00,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:02:00,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:02:00,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:02:00,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:02:03,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:02:05,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:10,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:13,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:02:15,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:15,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:02:15,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:02:15,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:02:15,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:02:20,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:02:20,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:23,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:02:25,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:30,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:02:30,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:02:30,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:02:30,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:02:31,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:31,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:02:36,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:41,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:02:41,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:45,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:02:45,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:02:45,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:02:45,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:02:47,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:49,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:02:49,406 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:02:49,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:02:49,409 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:02:49,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:02:50,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:02:51,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:02:52,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:02:57,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:00,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:03:00,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:03:00,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:03:00,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:03:01,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:03:03,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:08,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:10,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:03:14,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:15,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:03:15,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:03:15,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:03:15,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:03:19,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:20,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:03:20,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:03:25,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:30,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:30,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:03:30,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:03:30,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:03:30,909 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:03:31,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:03:36,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:39,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:03:41,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:47,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:03:47,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:03:47,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:03:47,099 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:03:49,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:03:50,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:03:52,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:03:57,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:01,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:04:02,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:04:02,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:04:02,097 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:04:03,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:08,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:11,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:04:13,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:17,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:04:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:04:17,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:04:17,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:04:19,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:20,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:04:21,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:04:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:29,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:04:30,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:32,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:04:32,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:04:32,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:04:35,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:39,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:04:41,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:46,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:47,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:04:47,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:04:47,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:04:47,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:04:49,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:04:50,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:04:51,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:56,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:04:59,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:05:02,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:02,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:05:02,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:05:02,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:05:07,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:09,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:05:12,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:05:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:05:17,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:05:18,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:19,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:05:20,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:05:24,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:29,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:05:30,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:32,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:05:32,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:05:32,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:05:35,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:40,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:05:40,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:45,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:47,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:05:47,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:05:47,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:05:50,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:05:50,335 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:05:51,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:56,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:05:59,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:06:01,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:02,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:06:02,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:06:02,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:06:07,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:12,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:06:17,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:06:17,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:06:17,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:20,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:06:23,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:24,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:06:24,806 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:06:24,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:06:24,808 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:06:25,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:06:26,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:06:27,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:06:28,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:32,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:06:32,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:06:32,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:06:34,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:37,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:06:40,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:46,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:47,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:06:47,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:06:47,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:06:47,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:06:50,336 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:06:51,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:06:55,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:06:56,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:01,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:03,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:07:03,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:07:03,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:07:03,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:07:07,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:07,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:07:12,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:15,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:07:17,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:18,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:07:18,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:07:18,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:07:18,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:07:20,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:07:23,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:25,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:07:28,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:33,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:07:33,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:07:33,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:07:34,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:35,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:07:39,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:44,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:45,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:07:48,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:07:48,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:07:48,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:07:48,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:07:50,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:07:50,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:07:53,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:07:55,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:00,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:03,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:08:03,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:08:03,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:08:03,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:08:05,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:08:06,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:11,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:14,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:08:16,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:18,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:08:18,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:08:18,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:08:20,340 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:08:22,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:24,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:08:27,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:33,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:33,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:08:33,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:08:33,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:08:33,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:08:34,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:08:39,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:44,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:08:44,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:48,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:08:48,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:08:48,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:08:49,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:50,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:08:55,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:08:56,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:00,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:03,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:09:03,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:09:03,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:09:05,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:09,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:10,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:15,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:18,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:09:18,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:09:18,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:09:18,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:09:19,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:20,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:09:21,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:26,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:29,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:31,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:33,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:09:33,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:09:33,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:09:37,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:39,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:42,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:46,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:09:46,799 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:09:46,799 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:09:46,801 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:09:47,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:09:47,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:47,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:48,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:09:48,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:09:48,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:09:48,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:09:48,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:09:50,342 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:09:53,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:59,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:09:59,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:03,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:10:03,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:10:03,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:10:03,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:10:05,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:07,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:10,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:15,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:17,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:18,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:10:18,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:10:18,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:10:18,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:10:20,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:10:21,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:26,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:27,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:31,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:34,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:10:34,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:10:34,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:10:34,100 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:10:37,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:38,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:42,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:48,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:48,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:49,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:10:49,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:10:49,099 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:10:50,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:10:53,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:10:58,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:10:59,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:04,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:04,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:11:04,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:11:04,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:11:04,443 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:11:08,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:11:09,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:14,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:18,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:11:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:11:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:11:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:11:19,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:11:20,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:11:20,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:28,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:11:30,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:34,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:11:34,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:11:34,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:11:34,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:11:36,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:38,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:11:41,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:46,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:11:47,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:49,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:11:49,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:11:49,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:11:49,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:11:50,344 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:11:52,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:11:57,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:11:58,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:03,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:04,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:12:04,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:12:04,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:12:04,442 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:12:05,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:12:08,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:13,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:15,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:12:18,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:12:19,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:12:19,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:12:19,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:12:20,345 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:12:24,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:25,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:12:29,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:34,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:12:34,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:12:34,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:12:34,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:12:34,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:35,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:12:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:44,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:12:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:49,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:12:49,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:12:49,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:12:49,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:12:50,346 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:12:50,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:55,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:12:56,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:00,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:02,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:13:02,819 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:13:02,820 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:13:02,822 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:13:03,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:13:04,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:04,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:13:04,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:13:04,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:13:04,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:13:05,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:05,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:10,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:14,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:16,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:20,348 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:13:20,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:13:20,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:13:20,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:13:20,632 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:13:21,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:24,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:26,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:32,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:32,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:35,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:13:35,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:13:35,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:13:35,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:13:37,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:42,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:42,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:47,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:50,349 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:13:50,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:13:50,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:13:50,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:13:50,634 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:13:53,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:13:54,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:13:58,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:02,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:14:03,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:05,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:14:05,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:14:05,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:14:09,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:14,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:19,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:20,349 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:14:20,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:14:20,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:14:20,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:14:25,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:30,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:35,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:14:35,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:14:35,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:14:35,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:14:36,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:38,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:14:41,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:47,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:47,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:14:50,350 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:14:50,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:14:50,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:14:50,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:14:52,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:14:57,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:14:58,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:03,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:05,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:15:05,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:15:05,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:15:05,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:15:07,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:15:08,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:13,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:18,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:15:19,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:20,350 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:15:20,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:15:20,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:15:20,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:15:24,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:28,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:15:30,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:35,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:15:35,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:15:35,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:15:35,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:38,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:15:40,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:46,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:48,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:15:50,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:15:50,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:15:50,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:15:50,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:15:51,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:15:56,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:15:57,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:02,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:05,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:16:05,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:16:05,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:16:06,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:07,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:12,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:16,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:18,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:20,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:16:20,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:16:20,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:16:20,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:16:23,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:26,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:29,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:34,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:35,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:16:35,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:16:35,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:16:36,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:39,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:44,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:16:44,236 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:16:44,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:16:44,239 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:16:44,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:16:44,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:45,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:46,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:50,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:16:50,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:16:50,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:16:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:16:50,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:16:50,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:16:54,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:16:55,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:00,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:04,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:17:05,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:17:05,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:17:05,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:17:05,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:10,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:14,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:17:16,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:20,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:17:21,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:21,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:17:21,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:17:21,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:17:21,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:17:24,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:17:26,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:31,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:34,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:17:36,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:17:36,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:17:36,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:17:37,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:42,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:17:43,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:48,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:50,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:17:51,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:17:51,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:17:51,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:17:52,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:17:53,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:17:58,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:02,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:18:04,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:06,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:18:06,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:18:06,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:18:09,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:12,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:18:15,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:20,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:18:21,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:21,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:18:21,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:18:21,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:18:22,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:18:26,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:32,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:32,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:18:36,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:18:36,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:18:36,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:18:37,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:42,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:18:43,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:48,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:50,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:18:50,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:18:51,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:18:51,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:18:51,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:18:53,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:18:58,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:02,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:19:03,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:06,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:19:06,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:19:06,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:19:09,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:12,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:19:14,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:20,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:20,355 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:19:20,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:19:21,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:19:21,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:19:21,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:19:25,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:30,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:19:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:36,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:37,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:19:37,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:19:37,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:19:37,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:19:40,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:19:42,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:47,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:50,356 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:19:50,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:19:52,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:19:52,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:19:52,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:19:52,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:19:52,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:57,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:19:57,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:19:57,816 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:19:57,817 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:19:57,818 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:19:57,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:19:59,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:20:02,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:07,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:20:07,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:20:07,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:20:07,010 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:20:08,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:09,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:20:13,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:18,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:20,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:20:20,356 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:20:22,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:20:22,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:20:22,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:20:22,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:20:24,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:29,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:32,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:20:35,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:37,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:20:37,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:20:37,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:20:37,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:20:40,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:42,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:20:46,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:50,359 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:20:51,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:20:52,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:20:52,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:20:52,010 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:20:52,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:20:54,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:20:57,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:02,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:04,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:21:07,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:21:07,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:21:07,010 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:21:07,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:21:07,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:12,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:16,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:21:18,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:20,358 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:21:23,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:21:23,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:21:23,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:21:23,136 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:21:23,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:26,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:21:28,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:33,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:37,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:21:38,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:21:38,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:21:38,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:21:38,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:21:39,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:44,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:49,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:21:49,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:21:50,359 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:21:53,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:21:53,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:21:53,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:21:53,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:21:55,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:00,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:22:00,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:05,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:08,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:22:08,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:22:08,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:22:08,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:22:10,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:22:10,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:15,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:20,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:22:21,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:22,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:22:23,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:22:23,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:22:23,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:22:23,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:22:26,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:31,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:32,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:22:36,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:38,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:22:38,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:22:38,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:22:38,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:22:42,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:42,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:22:47,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:50,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:22:52,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:22:53,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:22:53,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:22:53,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:22:53,367 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:22:54,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:22:57,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:02,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:04,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:23:07,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:08,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:23:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:23:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:23:08,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:23:13,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:14,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:23:18,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:20,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:23:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:23:23,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:23:23,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:23:23,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:23,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:23:27,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:23:28,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:33,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:36,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:23:36,379 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:23:36,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:23:36,384 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:23:37,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:23:37,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:23:38,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:23:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:23:38,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:23:38,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:23:39,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:23:39,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:44,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:49,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:23:50,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:23:50,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:23:53,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:23:53,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:23:53,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:23:53,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:23:55,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:00,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:01,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:24:05,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:08,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:24:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:24:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:24:08,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:24:11,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:11,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:24:16,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:20,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:24:20,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:24:21,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:24:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:24:23,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:24:23,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:24:26,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:31,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:37,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:24:38,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:24:38,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:24:38,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:24:38,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:24:42,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:48,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:48,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:24:50,370 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:24:53,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:24:53,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:24:53,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:24:53,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:24:53,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:24:58,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:24:58,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:03,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:08,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:25:08,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:25:08,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:25:08,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:25:09,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:10,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:25:14,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:19,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:20,370 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:25:21,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:25:23,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:25:23,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:25:23,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:25:25,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:31,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:33,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:25:36,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:38,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:25:38,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:25:38,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:25:38,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:25:41,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:43,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:25:46,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:50,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:25:52,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:25:53,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:25:53,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:25:53,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:25:55,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:25:57,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:02,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:05,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:26:08,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:26:08,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:26:08,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:26:13,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:15,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:26:18,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:20,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:26:23,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:26:23,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:26:23,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:26:23,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:26:24,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:25,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:26:29,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:34,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:37,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:26:38,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:26:38,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:26:38,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:26:39,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:44,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:46,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:26:50,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:50,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:26:53,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:26:53,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:26:53,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:26:53,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:26:55,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:26:58,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:27:01,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:06,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:08,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:27:08,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:27:08,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:27:08,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:27:11,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:16,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:19,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:27:19,608 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:27:19,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:27:19,612 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:27:20,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:27:20,374 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:27:21,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:27:22,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:27:22,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:23,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:27:23,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:27:23,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:27:27,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:33,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:27:33,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:38,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:27:38,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:27:38,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:27:38,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:43,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:47,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:27:49,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:50,374 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:27:53,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:27:53,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:27:53,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:27:54,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:27:59,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:28:00,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:05,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:08,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:28:08,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:28:08,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:28:11,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:11,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:28:16,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:20,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:28:21,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:21,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:28:23,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:28:23,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:28:23,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:28:27,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:32,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:33,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:28:37,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:38,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:28:38,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:28:38,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:28:43,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:45,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:28:48,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:50,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:28:53,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:28:53,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:28:53,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:28:53,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:28:53,923 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:28:56,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:28:59,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:04,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:07,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:29:08,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:29:08,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:29:08,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:29:08,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:29:10,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:15,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:19,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:29:20,377 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:29:20,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:23,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:29:23,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:29:23,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:29:23,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:29:26,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:29,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:29:31,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:36,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:38,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:29:38,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:29:38,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:29:38,923 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:29:41,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:29:41,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:46,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:50,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:29:52,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:29:53,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:29:53,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:29:53,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:29:53,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:29:55,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:29:58,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:03,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:05,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:30:08,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:08,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:30:08,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:30:08,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:30:08,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:30:14,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:17,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:30:19,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:20,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:30:24,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:25,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:30:25,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:30:25,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:30:25,249 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:30:27,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:30:29,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:34,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:39,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:39,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:30:40,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:30:40,236 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:30:40,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:30:45,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:50,379 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:30:51,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:30:51,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:30:55,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:30:55,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:30:55,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:30:56,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:02,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:03,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:31:07,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:10,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:31:10,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:31:11,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:31:11,001 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:31:12,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:12,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:31:12,277 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:31:12,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:31:12,281 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:31:13,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:31:13,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:31:14,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:31:17,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:20,380 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:31:22,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:25,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:31:25,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:31:25,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:31:25,998 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:31:28,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:33,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:35,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:31:38,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:40,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:31:40,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:31:40,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:31:40,999 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:31:44,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:47,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:31:49,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:50,381 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:31:55,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:31:55,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:31:56,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:31:56,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:31:56,004 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:31:59,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:32:01,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:06,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:09,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:32:10,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:32:10,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:32:11,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:32:11,000 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:32:11,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:16,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:20,382 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:32:21,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:21,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:32:25,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:32:26,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:32:26,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:32:26,001 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:32:27,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:31,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:32:32,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:37,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:40,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:32:41,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:32:41,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:32:41,000 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:32:43,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:43,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:32:48,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:50,382 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:32:54,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:32:55,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:32:56,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:32:56,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:32:56,001 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:33:00,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:04,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:33:05,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:10,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:12,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:33:12,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:33:12,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:33:12,306 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:33:15,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:16,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:33:20,383 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:33:21,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:26,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:27,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:33:27,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:33:27,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:33:27,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:33:27,304 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:33:32,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:38,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:39,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:33:42,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:33:42,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:33:42,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:33:42,305 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:33:43,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:48,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:49,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:33:50,383 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:33:54,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:33:58,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:33:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:33:58,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:33:58,577 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:33:59,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:01,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:34:04,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:09,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:13,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:34:13,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:34:13,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:34:13,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:34:13,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:34:15,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:20,387 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:34:21,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:25,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:34:27,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:28,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:34:28,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:34:28,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:34:28,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:34:32,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:35,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:34:37,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:42,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:44,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:34:44,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:34:44,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:34:44,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:34:47,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:34:48,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:50,385 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:34:53,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:57,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:34:59,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:34:59,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:34:59,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:34:59,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:34:59,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:35:05,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:09,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:35:09,337 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:35:09,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:35:09,351 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:35:10,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:35:10,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:35:10,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:12,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:35:14,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:35:14,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:35:14,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:35:14,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:35:16,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:20,386 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:35:21,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:22,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:35:26,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:30,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:35:30,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:35:30,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:35:30,660 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:35:31,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:34,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:35:36,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:41,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:44,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:35:45,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:35:45,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:35:45,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:35:45,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:35:47,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:50,389 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:35:53,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:35:56,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:35:58,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:00,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:36:00,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:36:00,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:36:00,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:36:03,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:05,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:36:09,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:14,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:15,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:36:15,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:36:15,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:36:15,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:36:17,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:36:20,388 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:36:20,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:25,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:29,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:36:30,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:36:30,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:36:30,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:36:30,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:35,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:39,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:36:41,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:45,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:36:45,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:36:45,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:36:46,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:50,389 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:36:51,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:36:52,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:36:57,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:00,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:37:00,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:37:00,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:37:01,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:37:03,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:08,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:14,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:37:14,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:15,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:37:15,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:37:15,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:37:19,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:20,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:37:25,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:26,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:37:30,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:30,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:37:30,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:37:30,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:37:35,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:36,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:37:40,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:45,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:37:45,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:37:45,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:37:45,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:48,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:37:50,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:37:51,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:56,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:37:58,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:38:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:38:00,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:38:00,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:38:01,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:06,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:09,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:38:12,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:15,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:38:15,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:38:15,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:38:17,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:20,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:38:21,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:38:23,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:28,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:38:30,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:38:30,682 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:38:33,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:38:33,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:38,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:43,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:38:44,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:45,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:38:45,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:38:45,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:38:49,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:50,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:38:54,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:38:54,204 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:38:54,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:38:54,207 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:38:54,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:38:55,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:38:55,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:38:56,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:39:00,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:39:00,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:39:00,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:39:06,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:39:06,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:11,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:15,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:39:15,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:39:15,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:39:17,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:18,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:39:20,393 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:39:22,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:28,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:39:28,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:39:30,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:39:30,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:39:33,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:38,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:42,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:39:44,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:45,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:39:45,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:39:45,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:39:49,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:39:50,394 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:39:52,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:39:55,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:00,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:01,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:40:01,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:40:01,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:40:01,129 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:40:04,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:40:05,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:10,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:14,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:40:16,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:16,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:40:16,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:40:16,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:40:16,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:40:20,396 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:40:21,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:26,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:40:27,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:31,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:40:31,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:40:31,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:40:32,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:35,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:40:38,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:43,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:46,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:40:46,126 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:40:46,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:40:47,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:40:49,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:50,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:40:54,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:40:59,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:41:00,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:01,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:41:01,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:41:01,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:41:06,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:11,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:12,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:41:16,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:41:16,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:41:16,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:41:17,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:20,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:41:22,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:41:22,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:27,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:31,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:41:31,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:41:31,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:41:33,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:38,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:41:38,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:43,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:46,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:41:46,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:41:46,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:41:46,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:41:49,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:41:50,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:41:50,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:41:55,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:01,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:01,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:42:01,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:42:01,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:42:01,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:42:02,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:42:06,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:12,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:42:12,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:16,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:42:16,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:42:16,586 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:42:17,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:20,404 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:42:23,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:24,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:42:28,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:32,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:42:32,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:42:32,228 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:42:34,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:34,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:42:39,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:44,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:45,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:42:45,302 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:42:45,303 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:42:45,305 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:42:45,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:42:46,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:42:47,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:42:47,223 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:42:47,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:42:50,400 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:42:50,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:55,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:42:58,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:43:00,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:02,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:43:02,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:43:02,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:43:06,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:07,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:43:11,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:16,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:17,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:43:17,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:43:17,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:43:17,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:43:20,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:43:20,400 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:43:22,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:27,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:30,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:43:32,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:43:32,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:43:32,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:43:32,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:37,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:42,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:43:42,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:47,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:43:47,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:43:47,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:43:48,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:50,404 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:43:54,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:43:54,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:43:59,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:02,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:44:02,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:44:02,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:44:02,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:44:05,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:06,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:44:10,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:16,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:16,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:44:17,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:44:17,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:44:17,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:44:17,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:44:20,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:44:21,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:27,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:28,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:44:32,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:32,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:44:32,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:44:32,559 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:44:32,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:44:37,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:38,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:44:42,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:47,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:47,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:44:47,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:44:47,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:44:47,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:44:50,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:44:50,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:44:52,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:44:57,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:02,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:45:02,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:45:02,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:45:02,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:45:02,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:45:02,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:07,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:12,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:45:13,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:17,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:45:17,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:45:17,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:45:17,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:45:18,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:20,404 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:45:24,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:24,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:45:29,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:32,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:45:32,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:45:32,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:45:32,797 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:45:35,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:37,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:45:40,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:45,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:46,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:45:47,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:45:47,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:45:47,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:45:47,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:45:50,404 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:45:50,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:55,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:45:58,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:01,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:02,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:46:02,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:46:02,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:46:02,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:46:06,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:08,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:12,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:17,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:17,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:46:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:46:17,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:46:17,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:46:20,405 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:46:20,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:22,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:27,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:30,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:46:30,139 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:46:30,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:46:30,143 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:46:30,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:46:30,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:31,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:32,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:46:32,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:46:32,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:46:32,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:46:33,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:38,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:42,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:43,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:47,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:46:47,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:46:47,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:46:47,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:46:48,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:50,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:46:54,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:46:54,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:46:59,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:02,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:47:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:47:02,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:47:02,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:47:04,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:47:04,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:09,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:15,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:16,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:47:17,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:47:17,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:47:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:47:17,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:47:20,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:47:21,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:26,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:47:27,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:32,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:32,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:47:32,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:47:32,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:47:32,805 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:47:37,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:41,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:47:42,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:47,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:47:47,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:47:47,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:47:47,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:47:47,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:50,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:47:53,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:47:53,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:47:58,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:02,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:48:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:48:02,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:48:02,809 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:48:03,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:05,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:48:08,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:13,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:48:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:48:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:48:17,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:48:19,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:48:19,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:20,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:48:25,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:28,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:48:31,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:32,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:48:32,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:48:32,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:48:32,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:48:36,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:40,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:48:42,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:47,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:48:47,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:48:47,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:48:47,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:48:47,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:50,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:48:52,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:48:53,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:48:58,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:02,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:49:02,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:49:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:49:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:49:02,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:49:04,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:09,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:14,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:14,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:49:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:49:17,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:49:17,869 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:49:17,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:49:20,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:20,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:49:25,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:26,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:49:30,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:32,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:49:32,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:49:32,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:49:32,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:49:36,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:39,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:49:41,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:46,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:47,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:49:47,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:49:47,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:49:47,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:49:50,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:49:51,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:49:51,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:49:56,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:01,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:50:01,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:02,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:50:02,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:50:02,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:50:02,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:50:07,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:12,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:13,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:50:17,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:17,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:50:17,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:50:17,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:50:17,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:50:20,419 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:50:22,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:22,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:50:22,866 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:50:22,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:50:22,869 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:50:23,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:50:25,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:50:27,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:32,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:33,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:50:33,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:50:33,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:50:33,024 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:50:37,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:50:38,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:43,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:46,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:50:48,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:50:48,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:50:48,019 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:50:49,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:50,411 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:50:54,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:50:58,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:51:00,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:03,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:51:03,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:51:03,019 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:51:05,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:10,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:51:11,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:16,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:18,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:51:18,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:51:18,022 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:51:20,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:51:20,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:51:21,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:26,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:31,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:32,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:51:33,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:51:33,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:51:33,020 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:51:37,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:42,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:45,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:51:47,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:48,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:51:48,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:51:48,019 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:51:50,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:51:52,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:51:55,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:51:57,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:02,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:03,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:52:03,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:52:03,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:52:03,946 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:52:07,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:52:08,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:13,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:18,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:18,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:52:18,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:52:18,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:52:18,943 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:52:19,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:52:20,413 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:52:23,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:28,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:29,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:52:33,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:33,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:52:33,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:52:33,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:52:33,943 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:52:38,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:41,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:52:43,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:48,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:48,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:52:48,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:52:48,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:52:48,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:52:50,413 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:52:53,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:52:54,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:52:59,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:03,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:53:03,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:53:03,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:53:03,948 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:53:04,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:53:05,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:10,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:15,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:16,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:53:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:53:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:53:18,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:53:20,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:20,414 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:53:25,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:29,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:53:31,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:33,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:53:33,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:53:33,950 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:53:36,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:41,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:53:41,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:46,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:48,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:53:48,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:53:48,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:53:50,415 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:53:51,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:53:52,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:53:57,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:01,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:54:03,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:03,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:54:03,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:54:03,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:54:09,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:09,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:54:09,149 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:54:09,150 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:54:09,152 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:54:09,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:54:11,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:54:14,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:18,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:54:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:54:18,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:54:19,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:20,415 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:54:24,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:29,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:33,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:54:33,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:54:33,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:54:33,953 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:54:35,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:40,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:45,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:54:46,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:48,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:54:48,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:54:48,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:54:50,416 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:54:51,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:54:55,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:54:57,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:02,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:03,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:55:03,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:55:03,953 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:55:05,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:55:08,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:13,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:17,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:55:18,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:55:18,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:55:18,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:55:20,416 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:55:24,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:27,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:55:29,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:34,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:34,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:55:34,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:55:34,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:55:34,737 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:55:36,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:55:39,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:44,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:49,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:55:49,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:55:49,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:55:49,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:55:49,733 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:55:50,418 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:55:50,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:55,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:55:59,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:56:01,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:04,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:56:04,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:56:04,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:56:04,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:56:06,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:09,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:56:12,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:17,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:19,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:56:19,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:56:19,787 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:56:20,419 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:56:21,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:56:22,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:27,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:31,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:56:33,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:34,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:56:34,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:56:34,739 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:56:38,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:41,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:56:44,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:49,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:50,426 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:56:50,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:56:50,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:56:50,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:56:50,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:56:53,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:56:54,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:56:59,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:05,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:05,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:57:05,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:57:05,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:57:05,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:57:10,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:15,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:57:16,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:20,420 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:57:20,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:57:20,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:57:20,742 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:57:21,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:27,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:27,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:57:32,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:36,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:57:36,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:57:36,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:57:36,354 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:57:38,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:40,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:57:43,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:48,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:50,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:57:51,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:57:51,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:57:51,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:57:51,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:57:54,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:57:59,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:00,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 20:58:00,756 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 20:58:00,757 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 20:58:00,760 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 20:58:01,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 20:58:01,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:58:02,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:58:04,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:06,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:58:06,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:58:06,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:58:10,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:13,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:58:15,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:20,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:58:21,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:58:21,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:58:21,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:58:21,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:26,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:29,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:58:31,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:36,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:58:36,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:58:36,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:58:36,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:58:36,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:39,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:58:42,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:47,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:49,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:58:50,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:58:51,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:58:51,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:58:51,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:58:52,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:58:57,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:01,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:59:03,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:08,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:08,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:59:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:59:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:59:08,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:59:11,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:59:13,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:18,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:20,423 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:59:21,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:59:23,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:59:23,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:59:23,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:59:23,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:59:23,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:28,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:34,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:36,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:59:38,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:59:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:59:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:59:38,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:59:39,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:45,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:46,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 20:59:50,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:50,424 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 20:59:53,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 20:59:53,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 20:59:53,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 20:59:53,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 20:59:55,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 20:59:58,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:00:00,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:06,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:07,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:00:08,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:00:08,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:00:08,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:00:11,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:17,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:17,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:00:20,424 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:00:22,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:23,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:00:23,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:00:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:00:23,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:00:27,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:29,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:00:32,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:38,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:00:38,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:00:38,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:00:38,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:00:38,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:39,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:00:43,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:48,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:50,426 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:00:51,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:00:53,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:00:53,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:00:53,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:00:53,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:00:53,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:00:58,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:03,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:01:04,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:01:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:01:08,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:01:08,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:01:09,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:13,962 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:01:15,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:20,427 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:01:21,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:23,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:01:23,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:01:23,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:01:26,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:01:26,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:31,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:36,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:01:37,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:38,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:01:38,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:01:38,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:01:42,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:45,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:01:45,566 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:01:45,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:01:45,569 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:01:46,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:01:48,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:01:48,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:50,427 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:01:54,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:01:54,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:01:54,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:01:54,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:01:54,701 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:01:58,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:02:00,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:05,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:08,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:02:09,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:02:09,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:02:09,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:02:10,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:15,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:19,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:02:20,428 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:02:21,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:24,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:02:24,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:02:24,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:02:26,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:29,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:02:31,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:36,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:39,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:02:39,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:02:39,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:02:39,696 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:02:42,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:47,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:50,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:02:51,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:02:53,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:02:54,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:02:54,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:02:54,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:02:58,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:01,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:03:04,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:09,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:10,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:03:10,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:03:10,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:03:10,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:03:13,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:03:15,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:20,434 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:03:21,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:24,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:03:25,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:03:25,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:03:25,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:03:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:32,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:34,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:03:37,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:40,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:03:40,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:03:40,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:03:43,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:46,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:03:48,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:50,433 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:03:54,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:03:55,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:03:55,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:03:55,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:03:56,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:04:00,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:05,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:08,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:04:10,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:04:10,124 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:04:10,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:04:11,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:16,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:18,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:04:20,434 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:04:21,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:26,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:04:26,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:04:26,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:04:26,351 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:04:26,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:29,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:04:31,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:36,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:39,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:04:41,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:04:41,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:04:41,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:04:41,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:46,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:49,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:04:50,434 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:04:52,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:57,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:04:57,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:04:57,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:04:57,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:04:57,765 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:05:01,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:03,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:08,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:11,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:12,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:05:12,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:05:12,761 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:05:13,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:18,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:19,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:05:19,506 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:05:19,506 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:05:19,508 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:05:20,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:05:20,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:20,434 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:05:22,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:24,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:27,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:05:27,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:05:27,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:05:27,996 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:05:30,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:30,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:35,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:40,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:42,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:42,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:05:42,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:05:42,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:05:42,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:05:46,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:50,435 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:05:51,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:52,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:05:56,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:05:58,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:05:58,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:05:58,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:05:58,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:06:02,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:02,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:06:07,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:12,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:12,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:06:13,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:06:13,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:06:13,989 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:06:17,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:20,436 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:06:22,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:22,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:06:27,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:29,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:06:29,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:06:29,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:06:29,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:06:32,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:06:33,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:38,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:43,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:43,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:06:44,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:06:44,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:06:44,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:06:48,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:50,437 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:06:53,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:06:54,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:59,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:06:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:06:59,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:06:59,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:07:03,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:07:04,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:09,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:14,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:07:14,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:07:14,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:07:14,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:07:15,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:20,439 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:07:21,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:24,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:07:26,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:29,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:07:29,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:07:29,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:07:31,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:34,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:07:37,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:42,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:44,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:07:44,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:07:44,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:07:46,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:07:47,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:50,438 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:07:53,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:56,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:07:58,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:07:59,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:07:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:07:59,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:08:04,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:06,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:09,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:14,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:14,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:08:14,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:08:14,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:08:16,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:19,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:20,439 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:08:25,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:26,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:29,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:08:29,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:08:29,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:08:30,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:36,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:36,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:41,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:44,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:08:44,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:08:44,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:08:44,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:08:45,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:08:45,677 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:08:45,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:08:45,680 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:08:45,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:08:46,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:46,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:48,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:50,439 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:08:52,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:56,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:08:57,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:08:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:08:59,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:08:59,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:09:02,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:08,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:08,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:09:13,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:15,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:09:15,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:09:15,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:09:15,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:09:18,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:09:19,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:20,440 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:09:24,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:28,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:09:30,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:30,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:09:30,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:09:30,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:09:30,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:09:36,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:38,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:09:41,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:45,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:09:45,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:09:45,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:09:45,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:09:47,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:48,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:09:50,441 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:09:53,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:09:58,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:09:58,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:00,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:10:00,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:10:00,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:10:00,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:10:04,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:08,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:10:09,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:14,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:15,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:10:15,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:10:15,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:10:15,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:10:18,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:10:19,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:20,442 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:10:25,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:28,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:10:30,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:30,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:10:30,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:10:30,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:10:30,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:10:36,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:38,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:10:41,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:45,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:10:45,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:10:45,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:10:45,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:10:47,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:48,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:10:50,442 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:10:52,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:58,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:10:58,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:11:00,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:11:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:11:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:11:00,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:11:04,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:08,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:11:09,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:14,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:15,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:11:15,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:11:15,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:11:15,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:11:18,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:11:19,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:20,443 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:11:25,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:29,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:11:30,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:30,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:11:30,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:11:30,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:11:30,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:11:36,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:39,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:11:41,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:45,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:11:45,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:11:45,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:11:45,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:11:47,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:49,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:11:50,443 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:11:52,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:57,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:11:58,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:00,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:12:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:12:00,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:12:00,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:12:03,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:06,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:12:06,376 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:12:06,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:12:06,379 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:12:07,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:12:08,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:08,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:09,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:13,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:15,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:12:15,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:12:15,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:12:15,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:12:18,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:19,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:20,444 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:12:24,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:26,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:29,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:30,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:12:30,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:12:30,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:12:30,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:12:35,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:38,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:40,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:45,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:45,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:12:45,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:12:45,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:12:45,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:12:48,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:12:50,445 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:12:51,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:56,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:12:58,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:13:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:13:00,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:13:00,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:13:00,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:13:02,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:07,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:08,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:13:12,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:15,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:13:15,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:13:15,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:13:15,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:13:18,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:18,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:13:20,446 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:13:23,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:28,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:13:28,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:30,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:13:30,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:13:30,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:13:30,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:13:33,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:39,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:44,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:45,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:13:45,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:13:45,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:13:45,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:13:45,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:13:50,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:13:50,450 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:13:55,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:13:55,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:00,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:01,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:14:01,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:14:01,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:14:01,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:14:05,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:14:05,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:10,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:15,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:14:16,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:14:16,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:14:16,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:14:16,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:14:20,451 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:14:21,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:24,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:14:27,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:32,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:32,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:14:32,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:14:32,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:14:32,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:14:34,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:14:37,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:42,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:44,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:14:47,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:14:47,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:14:47,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:14:47,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:50,451 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:14:53,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:14:54,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:14:58,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:02,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:15:02,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:15:02,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:15:02,706 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:15:03,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:04,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:15:08,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:13,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:16,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:15:17,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:15:17,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:15:17,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:15:19,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:20,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:15:25,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:26,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:15:30,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:33,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:15:33,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:15:33,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:15:33,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:15:35,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:15:35,217 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:15:35,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:15:35,219 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:15:35,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:15:36,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:36,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:15:41,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:46,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:47,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:15:48,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:15:48,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:15:48,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:15:50,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:15:51,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:15:57,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:15:57,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:02,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:04,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:16:04,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:16:04,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:16:04,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:16:07,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:16:07,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:12,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:17,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:19,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:16:19,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:16:19,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:16:19,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:16:19,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:16:20,455 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:16:23,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:28,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:29,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:16:33,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:34,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:16:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:16:34,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:16:38,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:16:38,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:43,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:48,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:16:49,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:16:49,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:16:49,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:16:49,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:50,455 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:16:55,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:16:58,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:00,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:04,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:17:04,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:17:04,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:17:04,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:17:06,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:08,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:11,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:16,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:18,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:19,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:17:19,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:17:19,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:17:20,456 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:17:20,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:22,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:27,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:30,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:32,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:34,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:17:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:17:34,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:17:38,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:39,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:43,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:48,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:49,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:49,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:17:49,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:17:49,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:17:50,457 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:17:54,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:17:59,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:17:59,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:04,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:18:04,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:18:04,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:18:04,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:18:05,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:09,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:18:11,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:16,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:19,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:18:19,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:18:19,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:18:19,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:18:20,468 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:18:21,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:26,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:29,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:18:31,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:18:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:18:34,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:18:36,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:39,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:18:41,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:46,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:49,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:18:49,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:18:49,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:18:49,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:18:50,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:18:52,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:57,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:18:57,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:18:57,977 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:18:57,978 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:18:57,983 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:18:58,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:18:59,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:19:02,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:04,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:19:04,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:19:04,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:19:08,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:08,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:19:13,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:18,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:19:19,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:19:19,300 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:19:19,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:19:19,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:20,459 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:19:25,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:28,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:19:30,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:34,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:19:34,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:19:34,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:19:36,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:38,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:19:41,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:46,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:49,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:19:49,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:19:49,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:19:49,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:19:50,459 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:19:52,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:58,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:19:59,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:20:03,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:04,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:20:04,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:20:04,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:20:08,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:09,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:20:13,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:19,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:20:19,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:19,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:20:19,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:20:19,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:20:20,460 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:20:24,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:29,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:20:30,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:34,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:20:34,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:20:34,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:20:35,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:39,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:20:41,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:46,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:49,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:20:49,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:20:49,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:20:49,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:20:50,460 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:20:51,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:20:56,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:01,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:21:02,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:04,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:21:04,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:21:04,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:21:07,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:11,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:21:13,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:18,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:19,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:21:19,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:21:19,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:21:20,461 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:21:21,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:21:23,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:28,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:31,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:21:33,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:34,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:21:34,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:21:34,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:21:39,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:41,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:21:44,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:49,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:21:49,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:21:49,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:21:49,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:21:49,825 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:21:50,463 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:21:52,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:21:55,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:00,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:03,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:22:04,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:22:04,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:22:04,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:22:04,818 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:22:06,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:11,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:13,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:22:16,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:20,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:22:20,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:22:20,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:22:20,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:22:20,887 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:22:22,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:22,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:22:22,176 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:22:22,176 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:22:22,179 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:22:23,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:22:23,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:22:27,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:32,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:33,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:22:35,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:22:35,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:22:35,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:22:35,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:22:38,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:43,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:22:43,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:48,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:50,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:22:51,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:22:51,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:22:51,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:22:51,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:22:54,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:22:55,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:22:59,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:05,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:23:05,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:06,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:23:06,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:23:06,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:23:06,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:23:11,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:15,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:23:17,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:20,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:23:21,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:23:21,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:23:21,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:23:21,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:23:22,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:25,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:23:28,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:33,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:35,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:23:36,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:23:36,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:23:36,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:23:36,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:23:38,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:43,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:45,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:23:48,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:50,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:23:51,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:23:51,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:23:51,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:23:51,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:23:54,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:23:55,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:23:59,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:04,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:06,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:24:06,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:24:06,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:24:06,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:24:08,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:24:10,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:16,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:17,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:24:20,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:24:21,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:24:21,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:24:21,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:24:21,274 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:24:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:27,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:29,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:24:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:36,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:24:36,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:24:36,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:24:36,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:24:38,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:39,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:24:43,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:48,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:49,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:24:50,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:24:51,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:24:51,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:24:51,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:24:51,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:24:54,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:24:59,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:01,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:25:04,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:06,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:25:06,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:25:06,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:25:06,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:25:10,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:11,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:25:15,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:20,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:20,468 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:25:21,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:25:21,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:25:21,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:25:21,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:25:26,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:31,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:33,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:25:36,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:36,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:25:36,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:25:36,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:25:41,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:44,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:25:46,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:50,469 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:25:51,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:25:51,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:25:51,893 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:25:51,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:25:52,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:25:52,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:25:52,075 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:25:52,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:25:52,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:25:52,078 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:25:54,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:25:57,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:03,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:04,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:26:06,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:26:06,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:26:06,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:26:08,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:13,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:14,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:26:18,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:20,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:26:22,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:26:22,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:26:22,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:26:22,419 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:26:24,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:25,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:26:29,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:35,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:35,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:26:37,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:26:37,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:26:37,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:26:40,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:45,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:26:45,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:50,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:26:51,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:26:52,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:26:52,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:26:52,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:26:52,730 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:26:55,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:26:56,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:01,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:05,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:27:06,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:07,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:27:07,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:27:07,726 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:27:11,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:15,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:27:17,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:20,472 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:27:22,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:23,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:27:23,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:27:23,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:27:25,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:27:27,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:32,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:35,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:27:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:27:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:27:38,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:27:38,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:43,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:45,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:27:48,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:50,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:27:53,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:27:53,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:27:53,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:27:53,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:27:54,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:27:59,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:04,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:28:05,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:08,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:28:08,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:28:08,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:28:08,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:28:10,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:14,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:28:16,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:20,474 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:28:22,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:23,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:28:23,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:28:23,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:28:24,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:28:27,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:32,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:34,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:28:37,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:38,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:28:38,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:28:38,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:28:43,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:44,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:28:48,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:50,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:28:53,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:53,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:28:53,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:28:53,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:28:58,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:28:59,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:29:03,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:08,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:29:08,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:29:08,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:29:09,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:29:09,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:14,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:17,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:29:17,817 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:29:17,817 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:29:17,819 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:29:18,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:29:19,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:29:19,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:20,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:29:23,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:29:23,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:29:23,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:29:23,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:29:24,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:29,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:29:30,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:35,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:38,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:29:38,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:29:38,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:29:39,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:29:40,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:45,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:49,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:29:50,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:29:51,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:53,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:29:53,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:29:53,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:29:56,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:29:59,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:30:02,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:08,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:08,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:30:08,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:30:08,644 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:30:10,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:30:13,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:19,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:20,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:30:20,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:30:23,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:30:23,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:30:23,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:30:24,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:30,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:30:30,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:35,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:38,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:30:38,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:30:38,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:30:40,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:30:41,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:46,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:50,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:30:50,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:30:52,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:30:53,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:30:53,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:30:53,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:30:57,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:00,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:31:03,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:08,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:31:08,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:31:08,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:31:08,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:10,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:31:13,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:18,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:20,477 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:31:23,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:31:23,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:31:23,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:31:24,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:29,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:34,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:37,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:31:38,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:31:38,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:31:38,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:31:39,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:44,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:49,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:31:50,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:31:50,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:53,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:31:53,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:31:53,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:31:55,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:31:59,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:32:01,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:06,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:08,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:32:08,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:32:08,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:32:10,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:32:11,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:16,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:20,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:32:22,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:32:22,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:23,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:32:23,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:32:23,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:32:27,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:32,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:32:33,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:38,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:39,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:32:39,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:32:39,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:32:39,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:32:42,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:32:43,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:48,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:50,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:32:52,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:32:54,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:32:54,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:32:54,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:32:54,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:32:54,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:32:59,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:00,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:33:00,386 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:33:00,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:33:00,389 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:33:00,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:33:02,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:33:05,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:09,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:33:09,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:33:09,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:33:09,503 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:33:10,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:12,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:33:15,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:20,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:33:21,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:24,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:33:24,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:33:24,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:33:24,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:33:26,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:32,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:32,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:33:37,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:39,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:33:39,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:33:39,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:33:39,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:33:42,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:42,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:33:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:50,480 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:33:51,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:33:52,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:33:54,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:33:54,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:33:54,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:33:54,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:33:57,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:01,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:34:03,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:08,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:09,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:34:09,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:34:09,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:34:09,502 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:34:12,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:34:14,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:19,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:20,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:34:22,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:34:24,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:34:24,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:34:24,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:34:24,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:34:24,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:29,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:32,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:34:35,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:39,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:34:39,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:34:39,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:34:39,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:34:40,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:42,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:34:45,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:50,482 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:34:51,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:34:52,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:34:54,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:34:54,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:34:54,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:34:56,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:01,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:02,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:35:06,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:09,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:35:09,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:35:09,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:35:12,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:35:12,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:17,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:20,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:35:22,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:35:22,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:24,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:35:24,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:35:24,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:35:27,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:32,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:35:33,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:38,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:39,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:35:39,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:35:39,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:35:42,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:35:43,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:48,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:50,485 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:35:52,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:35:54,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:35:54,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:35:54,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:35:54,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:35:59,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:02,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:36:05,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:10,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:36:10,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:36:10,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:36:10,138 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:36:11,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:12,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:36:16,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:20,485 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:36:22,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:22,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:36:25,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:36:25,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:36:25,137 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:36:25,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:36:27,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:31,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:36:31,066 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:36:31,067 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:36:31,068 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:36:32,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:36:32,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:36:33,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:38,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:40,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:36:40,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:36:40,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:36:40,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:36:42,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:36:43,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:48,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:50,486 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:36:52,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:36:53,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:36:55,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:36:55,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:36:55,137 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:36:59,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:02,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:37:05,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:10,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:37:10,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:37:10,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:37:10,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:12,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:37:16,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:20,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:37:21,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:22,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:37:25,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:37:25,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:37:25,139 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:37:27,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:32,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:37:33,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:38,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:40,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:37:40,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:37:40,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:37:42,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:37:43,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:48,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:50,488 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:37:53,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:37:54,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:37:55,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:37:55,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:37:55,138 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:37:59,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:02,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:38:04,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:09,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:10,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:38:10,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:38:10,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:38:10,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:38:12,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:38:15,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:20,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:20,490 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:38:24,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:38:25,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:38:25,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:38:25,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:38:25,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:30,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:34,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:38:36,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:40,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:38:40,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:38:40,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:38:40,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:38:41,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:45,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:38:47,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:50,491 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:38:52,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:38:55,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:38:55,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:38:55,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:38:55,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:38:57,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:02,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:05,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:39:07,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:10,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:39:10,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:39:10,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:39:10,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:39:13,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:14,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:39:18,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:20,492 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:39:24,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:25,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:39:25,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:39:25,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:39:26,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:39:29,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:35,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:36,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:39:40,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:40,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:39:40,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:39:40,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:39:45,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:46,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:39:50,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:50,492 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:39:55,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:39:55,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:39:55,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:39:55,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:39:55,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:39:56,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:39:56,477 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:39:56,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:39:56,481 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:39:56,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:39:58,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:40:00,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:40:00,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:05,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:10,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:40:10,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:40:10,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:40:11,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:16,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:20,493 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:40:22,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:25,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:40:25,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:40:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:40:28,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:33,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:38,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:40,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:40:40,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:40:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:40:43,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:48,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:50,493 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:40:53,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:40:55,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:40:55,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:40:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:40:59,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:04,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:09,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:10,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:41:10,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:41:10,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:41:10,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:41:14,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:19,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:20,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:41:25,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:41:25,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:41:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:41:25,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:30,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:35,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:40,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:41:40,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:41:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:41:41,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:46,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:50,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:41:51,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:41:55,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:41:55,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:41:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:41:57,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:02,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:07,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:10,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:42:10,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:42:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:42:13,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:18,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:20,507 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:42:23,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:25,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:42:25,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:42:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:42:29,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:34,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:39,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:40,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:42:40,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:42:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:42:45,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:50,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:42:50,508 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:42:55,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:42:55,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:42:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:42:56,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:01,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:06,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:10,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:43:10,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:43:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:43:12,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:17,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:20,508 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:43:22,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:25,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:43:25,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:43:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:43:27,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:32,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:37,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:40,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:43:40,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:43:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:43:43,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:48,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:50,510 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:43:53,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:43:55,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:43:55,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:43:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:43:59,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:04,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:09,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:10,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:44:10,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:44:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:44:14,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:19,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:20,509 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:44:24,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:25,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:44:25,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:44:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:44:30,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:35,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:40,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:44:40,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:44:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:44:40,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:45,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:50,510 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:44:51,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:44:55,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:44:55,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:44:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:44:57,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:02,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:07,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:10,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:45:10,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:45:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:45:12,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:17,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:20,511 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:45:23,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:45:25,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:45:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:45:29,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:34,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:39,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:40,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:45:40,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:45:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:45:44,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:49,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:45:50,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:45:55,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:45:55,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:45:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:45:55,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:00,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:05,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:10,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:46:10,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:46:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:46:10,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:15,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:20,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:20,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:46:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:46:25,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:46:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:46:25,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:30,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:35,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:40,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:46:40,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:46:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:46:41,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:46,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:50,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:46:51,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:46:55,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:46:55,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:46:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:46:57,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:02,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:07,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:10,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:47:10,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:47:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:47:12,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:17,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:20,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:47:23,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:25,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:47:25,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:47:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:47:25,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:47:28,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:33,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:38,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:40,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:47:40,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:47:40,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:47:44,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:49,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:50,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:47:54,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:47:55,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:47:55,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:47:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:47:59,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:04,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:09,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:10,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:48:10,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:48:10,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:48:15,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:20,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:48:21,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:25,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:48:25,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:48:25,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:48:26,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:31,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:36,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:40,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:48:40,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:48:40,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:48:42,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:47,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:50,516 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:48:52,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:48:55,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:48:55,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:48:55,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:48:58,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:03,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:08,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:10,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:49:10,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:49:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:49:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:49:14,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:19,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:20,516 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:49:24,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:25,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:49:25,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:49:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:49:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:34,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:39,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:40,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:49:40,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:49:40,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:49:45,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:50,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:49:50,517 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:49:55,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:49:55,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:49:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:49:56,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:01,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:06,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:10,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:50:10,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:50:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:50:11,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:16,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:20,517 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:50:21,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:25,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:50:25,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:50:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:50:26,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:31,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:36,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:40,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:50:40,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:50:40,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:50:42,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:47,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:50,518 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:50:52,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:50:55,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:50:55,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:50:55,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:50:55,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:50:57,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:02,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:07,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:10,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:51:10,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:51:10,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:51:13,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:18,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:20,518 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:51:23,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:25,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:51:25,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:51:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:51:29,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:34,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:39,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:40,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:51:40,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:51:40,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:51:44,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:47,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:51:47,016 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:51:47,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:51:47,020 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:51:47,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:51:47,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:51:49,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:51:50,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:51:50,534 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:51:55,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:51:55,298 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:51:55,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:51:55,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:00,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:05,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:10,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:52:10,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:52:10,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:52:11,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:13,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:52:16,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:20,520 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:52:22,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:25,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:52:25,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:52:25,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:52:25,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:52:27,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:32,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:37,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:52:40,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:52:40,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:52:40,817 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:52:43,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:48,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:50,520 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:52:53,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:55,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:52:56,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:52:58,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:52:59,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:52:59,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:52:59,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:52:59,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:53:01,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:53:03,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:07,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:53:08,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:13,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:18,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:20,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:53:22,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:53:22,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:53:22,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:53:23,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:25,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:53:28,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:29,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:53:33,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:37,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:53:37,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:53:39,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:44,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:53:44,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:49,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:50,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:53:52,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:53:52,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:53:54,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:53:59,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:53:59,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:04,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:07,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:54:07,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:54:10,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:14,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:54:15,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:20,522 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:54:21,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:22,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:54:22,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:54:26,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:29,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:54:31,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:33,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:54:37,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:37,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:54:37,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:54:42,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:43,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:54:44,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:54:47,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:50,522 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:54:53,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:54,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:54:54,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:54:54,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:54:57,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:54:59,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:54:59,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:55:04,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:07,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:55:09,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:55:09,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:55:09,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:55:09,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:14,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:16,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:55:19,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:20,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:55:20,523 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:55:24,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:55:24,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:55:24,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:55:24,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:30,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:55:30,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:31,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:55:35,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:39,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:55:39,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:55:39,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:55:40,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:55:40,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:45,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:46,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:55:50,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:55:50,524 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:55:51,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:55:54,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:55:54,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:55:54,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:55:56,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:00,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:56:01,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:56:02,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:07,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:09,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:56:09,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:56:09,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:56:12,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:56:12,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:16,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:56:17,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:20,527 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:56:22,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:56:23,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:24,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:56:24,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:56:24,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:56:28,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:31,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:56:31,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:56:34,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:39,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:39,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:56:39,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:56:39,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:56:41,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:56:44,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:46,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:56:49,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:50,527 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:56:51,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:56:54,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:56:54,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:56:54,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:56:54,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:56:59,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:01,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:57:03,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:57:05,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:09,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:57:09,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:57:09,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:57:10,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:14,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:57:16,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:16,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:57:20,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:57:21,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:24,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:57:24,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:57:24,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:57:24,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:57:26,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:31,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:33,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:57:36,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:57:37,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:39,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:57:39,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:57:39,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:57:42,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:47,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:48,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:57:48,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:57:50,529 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:57:53,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:57:54,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:57:54,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:57:54,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:57:58,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:00,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:03,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:58:03,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:08,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:09,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:58:09,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:58:09,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:58:09,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 21:58:09,624 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 21:58:09,631 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 21:58:09,640 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 21:58:09,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 21:58:11,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:12,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:14,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:18,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:58:19,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:20,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:58:21,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:24,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:58:24,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:58:24,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:58:25,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:30,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:33,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:58:33,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:36,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:40,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:58:40,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:58:40,227 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:58:41,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:43,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:46,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:48,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:58:50,531 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:58:51,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:58:53,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:58:55,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:58:55,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:58:55,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:58:57,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:02,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:03,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:59:06,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:59:07,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:10,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:59:10,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:59:13,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:18,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:20,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:59:20,534 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:59:23,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:24,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:59:25,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:59:25,178 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:59:29,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:34,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:59:35,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:59:35,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:40,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:41,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:59:41,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:59:41,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:59:44,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:59:45,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:50,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 21:59:50,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 21:59:51,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 21:59:56,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 21:59:56,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 21:59:56,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 21:59:56,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 21:59:56,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:01,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:05,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:00:06,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:00:07,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:12,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:12,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:00:12,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:00:12,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:00:16,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:00:18,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:20,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:00:20,536 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:00:23,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:26,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:00:27,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:00:27,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:00:27,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:00:28,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:33,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:35,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:00:36,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:00:39,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:43,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:00:43,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:00:43,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:00:45,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:45,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:00:50,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:00:50,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:50,536 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:00:55,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:00:58,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:00:58,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:00:58,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:01:01,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:05,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:01:06,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:08,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:01:11,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:15,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:01:15,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:01:15,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:01:16,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:18,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:01:20,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:01:20,537 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:01:22,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:27,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:28,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:01:30,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:01:30,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:01:30,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:01:33,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:36,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:01:38,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:40,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:01:43,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:45,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:01:45,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:01:45,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:01:48,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:01:48,356 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:01:48,356 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:01:48,359 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:01:48,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:01:49,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:01:50,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:01:50,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:01:51,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:01:54,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:00,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:00,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:02:00,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:02:00,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:02:00,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:02:05,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:07,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:02:10,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:02:11,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:15,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:02:15,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:02:15,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:02:16,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:20,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:02:20,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:02:22,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:02:22,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:27,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:30,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:02:30,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:02:30,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:02:30,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:02:32,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:37,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:37,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:02:40,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:02:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:45,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:02:45,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:02:45,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:02:48,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:02:50,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:02:50,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:02:52,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:02:54,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:00,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:00,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:03:00,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:03:00,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:03:01,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:03:05,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:08,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:03:10,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:03:11,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:15,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:03:15,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:03:15,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:03:16,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:20,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:03:20,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:03:22,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:23,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:03:27,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:30,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:03:30,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:03:30,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:03:30,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:03:33,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:38,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:39,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:03:42,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:03:43,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:45,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:03:45,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:03:45,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:03:48,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:50,540 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:03:52,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:03:54,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:03:54,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:03:59,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:00,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:04:00,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:04:04,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:04:04,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:09,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:11,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:04:14,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:04:15,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:04:15,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:04:15,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:20,552 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:04:21,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:26,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:04:26,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:04:27,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:30,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:04:30,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:04:32,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:36,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:04:38,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:41,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:04:43,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:45,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:04:45,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:04:48,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:04:49,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:50,545 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:04:54,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:04:56,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:05:00,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:00,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:05:00,696 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:05:01,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:05:05,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:11,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:05:11,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:13,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:05:15,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:05:15,696 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:05:16,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:20,548 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:05:22,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:22,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:05:22,008 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:05:22,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:05:22,013 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:05:22,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:05:24,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:05:25,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:05:26,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:05:27,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:31,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:05:31,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:05:31,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:05:32,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:34,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:05:37,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:42,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:05:42,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:46,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:05:46,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:05:46,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:05:48,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:50,551 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:05:54,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:05:57,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:05:58,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:06:00,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:01,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:06:01,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:06:05,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:08,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:06:11,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:12,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:06:16,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:16,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:06:16,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:06:18,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:06:20,548 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:06:21,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:26,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:27,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:06:28,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:06:31,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:06:31,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:06:31,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:36,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:40,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:06:42,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:42,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:06:47,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:06:47,446 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:06:47,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:50,548 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:06:50,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:06:53,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:06:57,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:06:58,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:01,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:07:02,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:07:02,444 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:07:03,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:08,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:11,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:07:12,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:07:14,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:17,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:07:17,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:07:19,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:20,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:07:21,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:07:25,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:28,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:07:31,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:07:31,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:32,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:07:32,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:07:36,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:41,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:07:41,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:43,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:07:46,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:47,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:07:47,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:07:50,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:07:51,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:52,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:07:56,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:07:58,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:08:02,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:08:02,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:02,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:08:02,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:08:07,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:12,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:08:13,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:08:13,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:17,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:08:17,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:08:18,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:20,553 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:08:24,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:24,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:08:28,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:08:29,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:32,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:08:32,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:08:34,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:08:34,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:39,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:43,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:08:44,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:08:45,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:47,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:08:47,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:08:50,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:08:51,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:52,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:08:52,772 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:08:52,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:08:52,775 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:08:52,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:08:54,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:08:56,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:08:58,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:09:01,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:02,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:09:02,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:09:05,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:09:07,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:12,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:13,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:09:17,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:09:17,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:09:17,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:09:17,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:20,553 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:09:23,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:27,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:09:28,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:09:29,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:09:32,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:09:34,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:37,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:09:40,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:44,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:09:45,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:47,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:09:47,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:09:47,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:09:50,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:09:51,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:56,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:09:59,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:09:59,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:10:01,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:02,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:10:02,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:10:07,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:09,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:10:12,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:14,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:10:17,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:17,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:10:17,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:10:20,555 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:10:20,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:10:23,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:28,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:10:30,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:10:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:10:32,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:10:33,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:38,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:40,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:10:44,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:44,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:10:48,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:10:48,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:10:50,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:10:50,555 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:10:53,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:10:55,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:00,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:11:00,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:03,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:11:03,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:11:03,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:11:06,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:15,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:11:15,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:11:16,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:18,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:11:18,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:11:20,558 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:11:21,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:25,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:11:26,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:30,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:11:31,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:33,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:11:33,909 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:11:35,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:11:37,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:42,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:45,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:11:47,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:11:47,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:48,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:11:48,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:11:50,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:11:53,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:11:57,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:11:59,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:00,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:12:04,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:04,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:12:04,946 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:12:07,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:12:10,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:15,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:15,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:12:18,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:12:19,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:12:19,947 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:12:20,558 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:12:20,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:25,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:28,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:12:28,414 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:12:28,415 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:12:28,418 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:12:28,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:12:30,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:12:30,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:12:31,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:34,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:12:34,945 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:12:37,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:41,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:12:42,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:45,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:12:47,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:50,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:12:50,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:12:50,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:12:52,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:12:53,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:12:57,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:00,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:13:02,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:03,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:13:05,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:13:05,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:13:08,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:13,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:13:14,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:15,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:13:19,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:20,560 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:13:20,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:13:20,705 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:13:23,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:13:24,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:29,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:30,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:13:33,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:13:35,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:35,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:13:35,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:13:40,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:45,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:13:45,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:13:45,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:50,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:13:51,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:13:52,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:13:52,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:13:52,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:13:55,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:13:57,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:00,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:14:02,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:05,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:14:07,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:14:07,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:14:07,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:14:07,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:12,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:15,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:14:17,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:14:17,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:20,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:14:23,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:23,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:14:23,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:14:23,758 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:14:27,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:14:28,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:30,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:14:33,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:37,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:14:38,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:14:38,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:14:38,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:14:38,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:43,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:45,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:14:45,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:14:49,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:14:50,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:14:53,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:14:53,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:14:53,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:14:54,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:00,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:00,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:15:01,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:15:05,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:08,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:15:08,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:15:08,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:15:10,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:11,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:15:15,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:15,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:15:20,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:20,563 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:15:21,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:15:23,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:15:23,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:15:23,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:15:25,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:30,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:15:31,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:31,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:15:36,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:38,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:15:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:15:38,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:15:41,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:15:41,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:45,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:15:46,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:50,563 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:15:51,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:15:52,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:53,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:15:53,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:15:53,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:15:57,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:15:59,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:15:59,017 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:15:59,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:15:59,020 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:15:59,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:16:00,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:16:01,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:16:03,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:08,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:08,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:16:08,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:16:08,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:16:11,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:16:13,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:15,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:16:18,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:20,564 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:16:21,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:16:23,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:23,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:16:23,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:16:23,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:16:28,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:30,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:16:31,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:16:34,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:39,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:16:39,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:16:39,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:16:40,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:42,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:16:45,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:45,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:16:50,565 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:16:51,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:16:51,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:16:54,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:16:54,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:16:54,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:16:57,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:00,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:17:01,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:17:03,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:08,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:09,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:17:09,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:17:09,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:17:11,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:17:13,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:15,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:17:18,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:20,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:17:21,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:17:24,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:17:24,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:17:24,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:29,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:30,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:17:31,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:17:34,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:39,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:17:39,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:17:40,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:41,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:17:45,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:45,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:17:50,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:17:50,567 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:17:53,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:17:54,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:17:54,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:17:55,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:00,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:02,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:18:05,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:18:06,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:09,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:18:09,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:18:11,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:15,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:18:16,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:17,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:18:20,567 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:18:22,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:24,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:18:24,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:18:25,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:18:27,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:32,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:18:32,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:38,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:18:38,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:39,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:18:39,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:18:43,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:47,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:18:48,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:18:48,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:18:50,567 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:18:54,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:18:54,417 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:18:54,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:00,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:19:00,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:02,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:19:05,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:09,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:19:09,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:19:09,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:19:10,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:15,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:17,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:19:19,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:19:20,568 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:19:21,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:25,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:19:25,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:19:25,838 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:19:27,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:27,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:19:27,301 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:19:27,302 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:19:27,304 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:19:27,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:19:29,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:19:32,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:19:32,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:38,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:39,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:19:40,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:19:40,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:19:40,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:19:44,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:47,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:19:47,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:19:49,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:50,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:19:55,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:19:56,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:19:56,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:19:56,138 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:19:59,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:20:01,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:02,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:20:06,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:09,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:20:11,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:20:11,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:20:11,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:20:12,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:17,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:20:17,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:17,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:20:20,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:20:23,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:26,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:20:26,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:20:26,147 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:20:27,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:20:29,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:32,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:20:34,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:37,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:20:39,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:41,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:20:41,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:20:41,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:20:45,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:47,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:20:48,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:20:50,570 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:20:50,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:55,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:20:56,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:20:56,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:20:56,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:20:58,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:21:01,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:02,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:21:06,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:08,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:21:11,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:21:11,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:21:11,089 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:21:12,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:17,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:21:17,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:18,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:21:20,570 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:21:22,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:26,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:21:26,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:21:26,090 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:21:28,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:28,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:21:32,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:21:33,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:38,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:21:38,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:41,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:21:41,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:21:41,089 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:21:44,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:47,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:21:48,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:21:49,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:50,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:21:54,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:21:56,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:21:56,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:21:56,090 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:21:56,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:00,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:03,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:22:05,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:05,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:10,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:11,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:22:11,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:22:11,093 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:22:15,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:16,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:18,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:22:20,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:22:21,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:25,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:26,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:22:26,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:22:26,089 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:22:27,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:32,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:33,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:22:35,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:38,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:42,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:22:42,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:22:42,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:22:43,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:43,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:22:43,844 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:22:43,844 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:22:43,847 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:22:44,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:22:45,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:48,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:22:48,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:50,588 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:22:54,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:22:55,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:22:57,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:22:57,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:22:57,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:22:59,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:03,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:23:05,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:05,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:23:10,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:12,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:23:12,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:23:12,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:23:16,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:23:16,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:18,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:23:20,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:23:21,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:24,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:23:26,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:27,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:23:27,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:23:27,639 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:23:32,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:33,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:23:34,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:23:37,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:42,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:23:42,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:23:42,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:23:43,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:44,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:23:48,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:23:48,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:50,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:23:54,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:23:54,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:23:57,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:23:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:23:57,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:23:59,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:03,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:24:04,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:24:04,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:09,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:12,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:24:12,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:24:12,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:24:14,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:24:15,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:18,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:24:20,578 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:24:21,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:24,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:24:26,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:27,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:24:27,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:24:27,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:24:32,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:32,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:24:33,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:24:37,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:41,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:24:42,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:24:42,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:24:42,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:24:42,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:47,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:48,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:24:50,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:24:53,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:24:53,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:24:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:24:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:24:57,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:24:58,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:03,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:25:03,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:25:04,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:09,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:12,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:25:12,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:25:12,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:25:13,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:25:14,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:18,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:25:19,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:20,580 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:25:24,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:25:25,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:27,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:25:27,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:25:27,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:25:30,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:33,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:25:34,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:25:36,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:41,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:42,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:25:42,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:25:42,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:25:44,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:25:46,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:48,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:25:50,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:25:52,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:54,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:25:57,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:25:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:25:57,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:25:57,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:26:02,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:02,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:26:02,690 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:26:02,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:26:02,692 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:26:03,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:26:03,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:26:04,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:26:07,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:12,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:26:12,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:26:12,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:26:12,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:14,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:26:17,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:18,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:26:20,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:26:23,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:24,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:26:27,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:26:27,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:26:27,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:26:29,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:33,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:26:34,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:26:35,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:40,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:42,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:26:42,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:26:42,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:26:44,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:26:46,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:48,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:26:50,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:26:51,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:54,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:26:57,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:26:57,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:26:57,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:26:57,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:27:02,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:03,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:27:04,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:27:08,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:12,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:27:12,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:27:12,696 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:27:14,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:14,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:27:18,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:27:19,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:20,585 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:27:24,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:24,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:27:27,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:27:27,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:27:27,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:27:29,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:33,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:27:33,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:27:35,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:40,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:42,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:27:42,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:27:42,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:27:44,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:27:45,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:48,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:27:50,586 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:27:51,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:54,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:27:57,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:27:57,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:27:57,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:27:57,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:28:03,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:03,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:28:04,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:28:08,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:12,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:28:12,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:28:12,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:28:13,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:14,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:28:18,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:28:18,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:20,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:28:24,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:24,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:28:27,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:28:27,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:28:27,643 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:28:29,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:33,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:28:34,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:28:35,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:40,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:43,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:28:43,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:28:43,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:28:46,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:28:46,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:48,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:28:50,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:28:52,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:56,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:28:58,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:28:58,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:28:58,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:28:58,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:29:03,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:03,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:29:06,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:29:09,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:13,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:29:13,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:29:13,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:29:14,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:16,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:29:18,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:29:19,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:20,588 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:29:24,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:29:24,647 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:29:24,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:29:24,650 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:29:24,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:29:25,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:25,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:29:26,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:29:28,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:29:28,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:29:28,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:29:30,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:29:35,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:36,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:29:40,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:43,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:29:43,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:29:43,516 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:29:46,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:48,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:29:50,589 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:29:51,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:29:52,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:57,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:29:58,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:29:58,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:29:58,518 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:30:00,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:30:03,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:03,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:30:08,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:10,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:30:13,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:13,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:30:13,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:30:18,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:30:18,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:30:18,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:20,589 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:30:24,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:28,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:30:28,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:30:28,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:30:29,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:30:34,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:38,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:30:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:43,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:30:43,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:30:45,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:48,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:30:50,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:30:50,590 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:30:51,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:56,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:30:58,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:30:58,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:30:58,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:31:00,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:31:02,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:03,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:31:07,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:10,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:31:13,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:13,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:31:13,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:31:18,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:18,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:31:20,601 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:31:20,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:31:24,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:28,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:31:28,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:31:28,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:31:30,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:32,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:31:33,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:31:35,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:40,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:42,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:31:43,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:31:43,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:31:43,811 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:31:46,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:49,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:31:50,591 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:31:51,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:53,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:31:56,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:31:59,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:31:59,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:31:59,791 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:32:01,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:03,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:04,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:32:06,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:12,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:13,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:14,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:32:14,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:32:14,790 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:32:17,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:19,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:32:20,592 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:32:22,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:23,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:27,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:30,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:32:30,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:32:30,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:32:32,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:33,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:34,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:32:38,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:42,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:44,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:45,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:32:45,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:32:45,719 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:32:49,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:32:50,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:32:50,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:32:51,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:32:51,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:32:51,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:32:51,679 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:32:52,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:32:52,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:54,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:32:55,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:00,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:00,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:33:00,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:33:00,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:33:02,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:33:05,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:33:06,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:12,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:12,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:33:15,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:33:15,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:33:15,719 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:33:17,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:20,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:33:20,594 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:33:22,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:33:23,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:28,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:30,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:33:30,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:33:30,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:33:32,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:33:33,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:35,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:33:38,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:40,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:33:44,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:45,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:33:45,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:33:45,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:33:49,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:33:50,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:33:50,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:33:52,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:33:55,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:00,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:00,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:34:00,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:34:00,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:34:02,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:34:05,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:34:05,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:10,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:13,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:34:15,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:34:15,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:34:15,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:34:16,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:20,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:34:20,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:34:22,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:23,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:34:27,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:30,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:34:30,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:34:30,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:34:32,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:33,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:34:35,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:34:37,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:43,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:34:43,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:45,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:34:45,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:34:45,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:34:48,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:50,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:34:50,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:34:53,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:34:54,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:34:59,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:00,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:35:00,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:35:00,723 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:35:03,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:35:04,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:05,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:35:09,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:12,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:35:15,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:15,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:35:15,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:35:15,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:35:20,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:35:20,599 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:35:20,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:22,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:35:25,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:30,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:35:30,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:35:30,723 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:35:30,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:32,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:35:35,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:35:36,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:41,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:42,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:35:45,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:35:45,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:35:45,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:35:46,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:50,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:35:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:35:52,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:35:54,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:35:57,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:01,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:36:01,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:36:01,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:36:02,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:04,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:36:06,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:36:07,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:12,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:36:12,306 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:36:12,306 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:36:12,309 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:36:12,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:36:13,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:14,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:36:16,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:36:16,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:36:16,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:36:18,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:20,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:36:21,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:36:24,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:25,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:36:29,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:31,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:36:31,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:36:31,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:36:35,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:35,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:36:36,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:36:40,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:45,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:36:45,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:46,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:36:46,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:36:50,602 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:36:51,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:36:51,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:36:55,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:36:56,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:01,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:37:01,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:37:01,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:37:02,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:05,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:37:06,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:37:07,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:13,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:15,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:37:16,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:37:16,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:37:19,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:20,603 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:37:21,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:37:24,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:25,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:37:29,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:32,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:37:32,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:37:32,152 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:37:35,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:35,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:37:36,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:37:40,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:44,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:37:45,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:47,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:37:47,150 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:37:50,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:37:51,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:37:52,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:37:54,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:37:57,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:02,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:02,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:38:02,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:38:02,854 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:38:04,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:38:07,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:38:08,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:13,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:16,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:38:17,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:38:17,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:38:17,853 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:38:18,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:20,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:38:22,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:38:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:29,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:38:29,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:32,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:38:32,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:38:32,853 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:38:35,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:37,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:38:39,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:38:40,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:45,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:47,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:38:47,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:38:47,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:38:49,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:38:50,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:38:50,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:38:52,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:38:55,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:01,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:01,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:02,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:39:02,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:39:02,853 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:39:07,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:08,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:39:12,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:13,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:17,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:17,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:39:17,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:39:17,856 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:39:20,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:39:23,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:23,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:23,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:39:28,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:32,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:39:32,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:39:32,853 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:39:33,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:34,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:38,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:39:39,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:42,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:39:42,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:39:42,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:39:42,678 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:39:42,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:39:44,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:44,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:45,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:47,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:39:47,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:39:47,854 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:39:50,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:39:50,608 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:39:53,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:39:55,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:39:55,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:00,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:02,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:40:02,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:40:02,854 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:40:06,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:08,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:40:11,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:16,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:17,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:40:17,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:40:17,896 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:40:20,608 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:40:21,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:23,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:40:26,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:40:27,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:32,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:34,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:40:34,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:40:34,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:40:37,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:40:38,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:38,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:40:43,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:47,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:40:48,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:49,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:40:49,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:40:50,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:40:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:40:54,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:40:57,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:40:59,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:04,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:41:04,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:41:04,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:41:05,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:07,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:41:09,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:41:10,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:15,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:17,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:41:19,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:41:19,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:41:19,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:41:20,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:41:21,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:24,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:41:27,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:41:27,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:32,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:34,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:41:34,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:41:34,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:41:37,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:41:38,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:39,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:41:43,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:47,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:41:49,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:49,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:41:49,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:41:50,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:41:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:41:54,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:41:57,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:41:59,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:04,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:42:04,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:04,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:42:04,746 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:42:07,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:42:09,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:42:10,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:15,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:17,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:42:19,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:42:19,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:42:20,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:42:20,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:24,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:42:26,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:27,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:42:31,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:34,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:42:34,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:42:34,705 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:42:37,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:38,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:42:39,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:42:42,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:47,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:42:47,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:49,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:42:49,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:42:50,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:42:52,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:42:54,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:42:57,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:42:57,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:02,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:04,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:43:04,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:43:04,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:43:07,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:43:08,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:09,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:43:13,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:17,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:43:19,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:19,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:43:19,705 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:43:20,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:43:24,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:25,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:43:27,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:43:27,036 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:43:27,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:43:27,038 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:43:27,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:43:29,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:43:30,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:35,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:36,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:43:36,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:43:36,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:43:39,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:43:40,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:40,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:43:45,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:49,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:43:50,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:50,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:43:51,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:43:51,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:43:56,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:43:56,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:43:59,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:44:01,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:06,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:06,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:44:06,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:44:06,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:44:09,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:44:11,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:44:12,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:17,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:19,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:44:20,614 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:44:21,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:44:21,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:44:21,923 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:44:23,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:27,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:44:28,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:29,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:44:33,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:37,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:44:37,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:44:37,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:44:39,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:40,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:44:42,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:44:44,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:49,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:50,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:44:50,614 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:44:52,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:44:52,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:44:52,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:44:55,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:44:57,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:45:00,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:45:00,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:05,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:08,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:45:08,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:45:08,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:45:10,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:12,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:45:12,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:45:15,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:20,614 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:45:21,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:23,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:45:23,985 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:45:24,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:45:27,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:45:27,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:27,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:45:32,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:37,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:45:38,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:38,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:45:38,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:45:44,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:44,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:45:47,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:45:49,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:50,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:45:54,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:45:54,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:45:54,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:45:54,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:45:59,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:45:59,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:46:00,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:05,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:09,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:46:09,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:46:09,996 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:46:10,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:14,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:46:15,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:19,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:46:20,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:20,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:46:25,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:26,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:46:26,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:46:26,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:46:29,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:46:29,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:46:30,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:35,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:40,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:46:40,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:46:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:46:41,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:46:44,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:46:46,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:50,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:46:50,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:46:52,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:56,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:46:56,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:46:56,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:46:57,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:46:59,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:47:02,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:03,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:08,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:09,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:47:09,488 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:47:09,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:47:09,491 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:47:10,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:47:11,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:11,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:47:11,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:47:12,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:13,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:14,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:47:18,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:20,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:47:22,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:24,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:28,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:47:28,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:47:28,005 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:47:29,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:47:30,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:31,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:35,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:39,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:41,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:43,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:47:43,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:47:43,004 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:47:44,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:47:46,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:49,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:47:50,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:47:51,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:56,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:47:58,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:47:58,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:47:58,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:47:59,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:47:59,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:48:01,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:06,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:09,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:48:11,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:13,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:48:13,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:48:13,005 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:48:14,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:48:17,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:20,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:48:20,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:48:22,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:27,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:28,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:48:28,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:48:28,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:48:29,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:48:30,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:48:32,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:37,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:40,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:48:43,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:48:43,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:48:43,005 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:48:43,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:44,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:48:48,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:50,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:48:52,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:48:53,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:58,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:48:59,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:48:59,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:48:59,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:48:59,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:49:02,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:49:04,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:09,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:13,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:49:14,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:49:14,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:49:14,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:49:19,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:20,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:49:21,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:49:25,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:29,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:49:29,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:49:29,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:49:29,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:49:30,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:31,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:49:36,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:41,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:41,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:49:44,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:49:44,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:49:44,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:49:44,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:49:46,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:50,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:49:51,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:49:52,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:49:57,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:00,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:50:00,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:50:00,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:50:00,150 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:50:02,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:04,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:07,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:12,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:13,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:15,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:50:15,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:50:15,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:50:15,149 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:50:19,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:20,622 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:50:22,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:25,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:30,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:30,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:50:30,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:50:30,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:50:30,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:50:31,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:50:31,198 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:50:31,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:50:31,200 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:50:31,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:50:32,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:33,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:40,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:42,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:45,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:50:45,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:50:45,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:50:45,149 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:50:45,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:50,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:50:50,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:50:54,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:50:55,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:00,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:01,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:51:01,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:51:01,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:51:01,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:51:04,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:51:06,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:11,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:14,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:51:16,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:51:16,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:51:16,032 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:51:17,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:20,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:51:22,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:26,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:51:28,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:32,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:51:32,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:51:32,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:51:32,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:51:33,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:35,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:51:39,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:44,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:47,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:51:47,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:51:47,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:51:47,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:51:49,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:50,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:51:55,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:51:57,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:52:01,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:02,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:52:02,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:52:02,684 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:52:06,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:08,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:52:12,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:17,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:17,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:52:17,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:52:17,684 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:52:18,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:52:20,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:52:22,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:28,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:28,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:52:32,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:52:32,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:52:32,685 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:52:33,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:38,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:52:39,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:44,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:47,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:52:47,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:52:47,684 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:52:48,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:52:49,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:50,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:52:55,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:52:58,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:53:01,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:02,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:53:02,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:53:02,687 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:53:06,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:10,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:53:11,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:16,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:17,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:53:17,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:53:17,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:53:17,687 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:53:20,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:53:20,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:53:22,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:27,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:30,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:53:32,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:53:32,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:53:32,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:53:32,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:37,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:40,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:53:43,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:47,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:53:47,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:53:47,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:53:47,761 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:53:48,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:50,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:53:50,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:53:54,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:53:59,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:53:59,362 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:53:59,363 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:53:59,366 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:54:00,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:54:00,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:00,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:02,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:02,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:54:02,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:54:02,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:54:02,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:54:05,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:10,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:11,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:16,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:18,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:54:18,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:54:18,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:54:18,906 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:54:20,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:54:21,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:22,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:26,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:31,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:32,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:33,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:54:33,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:54:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:54:33,905 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:54:37,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:40,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:42,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:47,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:48,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:54:48,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:54:48,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:54:48,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:54:50,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:54:52,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:54:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:54:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:02,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:55:03,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:55:03,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:55:03,905 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:55:03,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:55:04,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:09,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:12,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:55:14,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:19,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:19,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:55:19,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:55:19,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:55:19,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:55:20,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:55:22,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:55:25,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:30,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:32,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:55:34,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:55:34,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:55:34,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:55:36,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:41,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:42,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:55:46,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:50,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:55:50,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:55:50,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:55:50,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:55:50,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:55:52,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:55:53,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:55:57,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:02,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:03,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:56:05,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:56:05,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:56:05,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:56:08,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:13,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:14,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:56:18,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:20,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:56:20,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:56:20,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:56:20,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:56:20,798 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:56:24,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:56:24,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:29,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:34,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:56:34,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:35,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:56:35,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:56:35,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:56:35,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:56:39,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:44,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:56:45,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:50,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:56:51,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:56:52,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:56:52,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:56:52,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:56:52,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:56:54,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:56:57,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:02,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:04,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:07,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:57:07,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:57:07,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:57:07,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:57:08,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:13,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:14,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:20,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:57:22,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:57:22,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:57:22,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:57:22,452 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:57:23,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 22:57:23,807 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 22:57:23,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 22:57:23,810 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 22:57:24,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 22:57:24,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:24,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:25,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:29,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:34,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:35,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:37,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:57:37,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:57:37,449 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:57:40,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:44,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:46,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:48,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:50,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:57:51,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:57:52,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:57:52,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:57:52,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:57:52,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:57:54,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:57:57,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:02,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:07,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:58:07,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:07,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:58:07,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:58:07,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:58:13,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:17,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:58:18,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:20,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:58:23,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:58:23,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:58:23,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:58:23,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:58:24,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:27,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:58:30,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:35,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:36,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:58:38,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:58:38,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:58:38,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:58:41,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:46,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:58:46,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:50,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:58:52,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:55,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:58:55,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:58:55,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:58:55,136 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:58:57,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:58:58,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:59:02,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:07,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:08,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:59:10,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:59:10,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:59:10,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:59:10,135 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:59:13,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:18,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:59:19,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:20,637 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:59:24,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:26,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:59:26,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:59:26,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:59:26,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:59:28,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:59:29,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:34,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:38,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:59:40,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:42,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:59:42,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:59:42,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:59:42,107 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 22:59:46,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:50,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 22:59:50,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 22:59:51,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:56,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 22:59:57,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 22:59:57,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 22:59:57,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 22:59:57,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:00:01,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:01,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:06,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:11,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:12,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:12,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:00:12,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:00:12,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:00:12,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:00:17,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:20,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:00:21,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:23,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:28,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:00:28,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:00:28,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:00:28,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:00:28,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:31,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:34,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:39,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:43,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:00:43,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:00:43,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:00:43,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:00:44,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:47,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:49,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:50,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:00:55,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:00:55,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:00:55,216 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:00:55,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:00:55,219 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:00:55,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:00:56,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:57,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:00:58,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:00:58,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:00:58,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:01:00,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:06,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:01:06,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:11,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:13,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:01:13,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:01:13,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:01:13,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:01:16,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:01:16,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:20,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:01:22,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:24,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:01:28,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:01:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:01:28,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:01:28,905 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:01:33,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:34,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:01:38,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:43,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:01:43,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:01:43,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:01:44,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:44,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:01:49,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:50,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:01:54,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:01:57,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:01:58,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:01:58,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:01:58,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:01:58,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:02:00,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:05,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:09,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:02:10,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:13,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:02:13,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:02:13,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:02:13,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:02:16,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:19,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:02:20,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:02:21,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:26,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:28,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:02:28,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:02:28,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:02:28,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:02:29,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:02:32,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:37,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:39,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:02:42,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:43,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:02:43,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:02:43,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:02:43,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:02:48,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:49,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:02:50,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:02:53,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:58,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:02:58,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:02:58,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:02:58,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:02:59,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:02:59,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:03:04,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:09,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:03:09,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:13,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:03:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:03:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:03:13,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:03:15,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:18,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:03:20,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:03:20,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:25,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:28,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:03:28,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:03:28,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:03:28,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:03:28,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:03:31,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:36,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:40,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:03:42,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:43,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:03:43,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:03:43,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:03:43,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:03:47,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:50,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:03:50,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:03:52,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:57,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:03:58,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:03:58,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:03:58,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:03:58,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:04:01,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:02,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:07,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:11,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:13,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:04:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:04:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:04:13,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:04:19,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:20,644 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:04:20,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:04:20,856 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:04:20,857 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:04:20,858 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:04:21,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:04:22,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:23,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:24,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:29,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:30,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:04:30,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:04:30,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:04:30,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:04:33,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:35,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:40,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:43,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:45,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:04:45,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:04:45,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:04:45,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:50,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:04:50,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:04:53,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:04:55,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:00,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:00,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:05:00,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:05:00,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:05:00,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:05:03,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:05:05,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:10,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:13,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:05:15,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:05:15,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:05:15,780 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:05:16,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:20,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:05:21,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:23,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:05:26,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:31,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:05:31,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:05:31,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:05:31,298 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:05:32,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:33,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:05:37,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:42,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:45,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:05:46,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:05:46,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:05:46,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:05:47,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:50,648 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:05:52,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:05:54,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:05:57,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:01,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:06:01,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:06:01,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:06:03,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:05,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:06:08,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:13,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:16,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:06:16,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:06:16,297 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:06:17,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:06:19,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:20,648 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:06:24,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:27,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:06:30,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:31,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:06:31,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:06:31,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:06:35,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:37,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:06:40,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:45,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:46,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:06:46,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:06:46,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:06:47,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:06:50,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:06:51,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:56,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:06:59,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:01,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:07:01,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:07:01,295 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:07:02,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:07,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:11,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:12,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:17,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:07:17,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:07:17,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:07:17,739 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:07:17,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:19,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:20,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:07:23,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:29,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:29,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:32,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:07:32,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:07:32,735 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:07:34,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:39,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:40,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:45,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:47,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:07:47,618 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:07:47,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:07:47,621 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:07:47,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:07:47,735 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:07:47,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:07:47,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:07:47,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:07:48,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:49,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:07:50,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:07:50,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:55,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:07:59,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:08:01,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:08:02,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:08:02,735 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:08:06,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:09,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:08:12,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:17,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:08:17,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:08:17,735 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:08:17,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:19,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:08:20,651 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:08:23,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:28,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:08:29,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:32,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:08:32,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:08:32,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:08:32,737 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:08:34,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:37,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:08:39,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:44,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:47,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:08:47,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:08:47,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:08:47,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:08:50,651 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:08:50,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:55,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:08:57,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:09:00,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:04,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:09:04,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:09:04,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:09:04,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:09:06,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:07,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:09:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:17,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:17,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:09:19,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:09:19,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:09:19,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:09:19,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:09:20,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:09:22,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:25,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:09:27,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:32,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:34,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:09:34,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:09:34,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:09:34,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:09:38,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:43,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:48,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:50,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:09:50,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:09:50,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:09:50,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:09:50,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:09:53,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:09:54,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:09:59,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:03,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:10:04,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:05,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:10:05,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:10:05,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:10:05,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:10:09,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:13,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:10:14,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:19,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:20,654 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:10:20,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:10:20,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:10:20,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:10:20,864 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:10:23,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:10:24,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:29,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:33,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:10:35,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:35,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:10:35,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:10:35,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:10:35,862 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:10:41,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:41,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:10:46,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:50,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:10:50,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:10:50,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:10:50,861 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:10:50,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:10:51,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:10:52,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:10:56,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:01,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:02,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:11:05,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:11:05,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:11:05,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:11:05,862 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:11:07,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:12,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:11:12,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:17,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:19,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:11:19,766 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:11:19,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:11:19,769 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:11:20,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:11:20,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:11:20,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:11:20,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:11:20,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:11:20,863 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:11:21,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:11:23,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:28,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:31,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:11:33,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:35,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:11:35,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:11:35,862 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:11:35,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:11:39,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:41,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:11:44,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:50,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:11:50,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:11:50,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:11:50,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:11:50,862 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:11:50,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:11:51,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:11:56,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:01,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:01,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:12:05,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:12:05,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:12:05,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:12:05,866 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:12:07,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:09,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:12:13,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:18,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:19,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:12:20,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:12:20,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:12:20,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:12:20,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:12:20,863 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:12:24,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:29,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:29,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:12:34,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:35,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:12:35,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:12:35,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:12:35,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:12:39,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:39,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:12:44,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:50,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:12:50,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:12:50,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:12:50,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:12:50,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:12:50,863 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:12:50,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:12:56,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:00,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:13:01,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:06,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:13:06,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:13:06,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:13:06,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:13:07,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:10,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:13:12,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:18,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:20,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:13:20,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:13:21,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:13:21,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:13:21,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:13:21,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:13:23,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:28,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:13:29,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:34,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:36,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:13:36,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:13:36,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:13:36,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:13:38,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:13:39,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:44,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:48,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:13:50,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:50,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:13:51,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:13:51,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:13:51,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:13:55,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:13:57,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:00,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:06,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:14:06,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:14:06,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:14:06,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:07,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:11,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:16,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:17,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:20,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:14:21,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:14:21,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:14:21,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:14:22,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:27,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:28,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:33,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:36,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:14:36,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:14:36,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:14:36,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:14:37,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:14:37,458 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:14:37,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:14:37,461 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:14:37,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:14:37,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:38,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:38,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:43,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:49,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:49,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:14:50,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:14:51,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:14:51,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:14:51,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:14:54,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:14:59,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:00,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:15:04,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:06,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:15:06,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:15:06,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:15:06,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:15:10,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:12,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:15:15,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:20,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:20,661 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:15:21,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:15:21,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:15:21,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:15:22,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:15:26,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:31,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:34,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:15:36,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:15:36,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:15:36,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:15:37,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:42,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:47,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:48,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:15:50,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:15:51,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:15:51,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:15:51,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:15:53,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:58,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:15:59,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:16:03,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:07,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:16:07,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:16:07,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:16:07,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:16:09,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:09,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:16:14,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:19,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:20,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:16:21,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:16:22,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:16:22,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:16:22,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:16:22,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:16:25,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:30,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:31,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:16:36,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:37,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:16:37,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:16:37,336 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:16:41,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:44,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:16:46,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:50,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:16:52,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:16:52,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:16:52,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:16:52,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:16:52,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:16:57,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:02,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:17:02,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:07,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:17:07,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:17:07,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:17:08,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:12,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:17:14,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:19,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:20,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:17:22,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:17:22,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:17:22,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:17:22,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:17:24,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:29,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:32,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:17:34,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:37,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:17:37,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:17:37,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:17:37,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:17:39,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:42,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:17:45,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:50,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:17:50,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:17:52,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:17:52,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:17:52,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:17:52,355 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:17:54,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:17:56,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:01,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:05,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:18:06,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:07,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:18:07,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:18:07,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:18:07,400 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:18:12,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:14,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:18:14,416 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:18:14,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:18:14,418 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:18:14,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:18:15,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:18:18,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:20,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:18:22,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:18:22,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:18:22,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:18:22,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:18:23,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:25,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:18:28,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:33,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:18:34,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:37,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:18:37,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:18:37,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:18:37,683 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:18:39,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:44,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:18:45,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:50,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:18:50,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:18:52,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:18:52,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:18:52,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:18:52,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:18:54,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:18:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:00,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:04,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:19:06,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:07,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:19:07,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:19:07,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:19:07,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:19:11,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:14,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:19:16,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:20,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:19:22,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:22,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:19:22,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:19:22,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:19:22,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:19:22,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:19:27,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:32,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:19:33,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:38,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:38,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:19:38,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:19:38,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:19:38,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:19:40,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:19:43,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:48,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:19:50,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:19:50,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:19:53,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:19:53,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:19:53,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:19:53,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:19:54,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:00,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:00,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:20:05,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:08,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:20:08,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:20:08,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:20:08,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:20:10,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:15,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:20,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:20:20,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:20,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:20:23,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:20:23,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:20:23,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:20:23,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:20:25,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:28,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:20:31,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:36,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:38,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:20:38,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:20:38,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:20:38,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:20:38,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:20:42,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:48,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:48,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:20:50,670 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:20:53,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:20:54,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:20:54,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:20:54,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:20:54,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:20:56,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:20:58,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:03,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:06,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:09,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:09,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:21:09,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:21:09,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:21:09,838 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:21:14,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:16,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:19,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:20,673 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:21:24,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:24,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:24,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:21:24,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:21:24,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:21:24,838 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:21:30,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:33,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:21:33,626 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:21:33,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:21:33,629 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:21:34,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:21:34,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:35,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:35,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:39,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:21:39,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:21:39,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:21:39,838 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:21:41,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:44,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:46,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:50,673 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:21:51,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:21:54,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:21:54,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:21:54,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:21:54,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:21:54,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:21:57,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:02,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:22:02,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:10,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:22:10,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:22:10,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:22:10,326 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:22:12,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:22:13,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:18,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:20,673 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:22:22,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:22:23,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:25,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:22:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:22:25,324 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:22:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:22:29,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:30,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:22:34,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:40,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:40,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:22:40,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:22:40,324 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:22:40,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:22:45,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:50,674 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:22:50,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:22:50,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:22:55,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:22:55,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:22:55,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:22:55,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:22:56,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:00,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:07,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:08,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:10,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:23:10,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:23:10,325 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:23:12,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:17,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:18,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:20,674 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:23:23,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:25,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:23:25,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:23:25,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:23:29,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:29,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:34,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:37,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:39,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:40,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:23:40,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:23:40,325 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:23:44,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:47,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:49,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:50,675 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:23:54,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:23:55,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:23:55,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:23:55,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:23:55,325 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:24:00,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:04,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:05,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:10,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:11,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:24:11,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:24:11,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:24:11,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:24:14,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:16,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:20,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:24:21,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:24,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:26,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:24:26,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:24:26,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:24:26,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:24:27,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:32,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:34,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:37,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:41,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:24:41,692 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:24:41,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:24:41,695 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:24:41,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:24:41,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:24:41,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:24:42,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:24:42,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:43,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:43,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:48,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:50,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:24:52,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:24:53,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:24:56,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:24:56,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:24:56,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:24:58,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:02,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:25:04,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:09,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:11,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:25:11,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:25:11,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:25:12,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:25:14,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:20,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:25:20,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:20,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:25:25,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:28,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:25:28,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:25:28,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:25:28,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:25:30,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:25:31,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:36,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:41,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:25:42,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:43,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:25:43,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:25:43,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:25:43,020 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:25:47,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:50,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:25:51,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:25:52,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:58,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:25:58,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:25:58,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:25:58,020 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:25:58,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:25:59,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:03,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:08,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:09,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:13,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:26:13,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:26:13,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:26:13,020 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:26:14,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:17,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:19,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:20,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:26:24,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:28,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:26:28,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:26:28,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:26:28,020 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:26:29,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:30,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:35,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:36,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:40,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:43,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:26:43,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:26:43,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:26:43,024 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:26:45,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:46,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:50,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:26:51,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:56,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:26:57,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:26:58,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:26:58,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:26:58,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:26:58,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:27:02,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:04,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:08,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:13,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:13,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:27:13,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:27:13,333 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:27:13,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:27:16,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:18,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:20,679 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:27:24,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:24,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:28,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:27:28,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:27:28,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:27:29,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:34,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:35,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:40,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:43,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:27:43,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:27:43,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:27:44,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:46,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:50,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:27:51,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:52,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:27:52,637 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:27:52,637 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:27:52,639 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:27:52,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:27:53,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:54,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:27:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:27:58,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:27:58,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:27:58,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:28:02,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:03,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:28:07,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:13,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:13,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:28:13,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:28:13,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:28:13,331 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:28:18,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:20,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:28:21,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:28:23,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:28,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:28:28,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:28:28,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:28:28,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:28:28,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:31,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:28:34,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:40,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:41,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:28:43,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:28:43,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:28:43,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:28:45,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:50,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:28:51,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:28:51,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:56,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:28:58,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:28:58,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:28:58,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:28:59,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:01,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:06,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:09,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:12,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:13,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:29:13,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:29:13,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:29:17,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:18,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:20,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:29:23,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:26,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:28,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:29:28,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:29:28,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:29:28,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:29:28,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:33,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:36,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:39,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:43,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:29:43,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:29:43,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:29:43,938 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:29:45,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:46,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:50,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:50,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:29:55,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:29:56,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:29:58,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:29:58,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:29:58,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:29:58,938 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:30:01,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:04,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:30:07,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:12,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:13,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:30:13,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:30:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:30:13,937 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:30:17,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:19,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:30:20,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:30:22,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:28,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:28,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:30:28,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:30:28,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:30:28,937 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:30:29,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:30:34,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:37,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:30:39,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:43,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:30:43,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:30:43,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:30:43,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:30:45,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:47,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:30:50,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:50,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:30:55,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:30:57,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:30:58,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:30:58,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:30:58,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:30:58,937 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:31:01,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:05,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:31:05,186 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:31:05,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:31:05,189 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:31:05,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:31:06,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:06,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:07,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:11,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:31:13,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:31:13,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:31:13,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:31:15,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:16,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:20,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:31:21,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:25,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:26,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:28,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:31:28,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:31:28,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:31:28,938 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:31:32,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:35,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:37,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:42,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:43,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:31:43,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:31:43,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:31:43,938 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:31:45,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:48,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:50,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:31:53,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:55,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:31:58,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:31:58,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:31:58,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:31:58,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:31:58,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:32:03,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:32:04,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:09,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:13,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:32:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:32:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:32:13,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:32:13,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:32:14,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:19,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:20,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:32:22,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:32:24,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:29,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:32:29,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:32:29,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:32:29,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:32:30,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:30,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:32:35,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:40,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:40,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:32:44,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:32:44,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:32:44,052 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:32:45,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:50,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:50,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:32:51,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:32:55,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:32:59,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:32:59,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:32:59,052 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:32:59,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:33:01,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:06,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:09,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:33:11,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:15,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:33:15,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:33:15,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:33:15,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:33:17,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:19,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:33:20,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:33:22,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:27,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:33:28,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:30,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:33:30,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:33:30,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:33:33,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:37,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:33:39,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:44,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:45,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:33:45,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:33:45,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:33:47,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:33:50,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:50,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:33:55,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:33:57,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:00,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:34:00,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:34:00,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:34:01,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:05,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:06,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:11,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:13,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:34:13,403 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:34:13,404 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:34:13,406 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:34:13,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:34:14,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:15,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:34:15,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:34:15,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:34:15,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:34:15,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:17,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:20,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:34:23,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:25,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:28,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:30,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:34:30,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:34:30,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:34:30,938 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:34:33,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:33,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:38,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:43,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:44,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:45,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:34:45,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:34:45,934 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:34:50,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:34:50,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:34:53,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:34:55,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:00,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:35:00,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:35:00,934 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:35:01,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:01,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:06,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:11,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:11,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:15,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:35:15,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:35:15,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:35:15,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:35:17,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:20,689 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:35:22,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:22,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:27,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:29,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:30,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:35:30,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:35:30,934 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:35:33,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:38,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:39,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:43,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:46,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:35:46,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:35:46,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:35:46,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:35:49,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:49,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:50,689 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:35:54,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:35:59,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:35:59,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:36:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:36:01,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:36:05,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:07,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:36:11,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:16,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:36:16,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:36:16,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:36:16,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:17,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:36:20,690 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:36:21,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:26,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:27,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:36:31,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:36:31,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:36:31,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:36:31,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:36:32,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:35,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:36:38,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:43,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:45,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:36:46,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:36:46,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:36:46,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:36:49,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:50,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:36:55,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:36:55,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:00,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:01,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:37:01,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:37:01,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:37:01,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:37:03,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:05,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:10,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:13,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:15,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:16,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:37:16,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:37:16,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:37:20,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:37:20,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:21,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:37:21,036 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:37:21,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:37:21,039 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:37:21,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:37:22,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:23,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:26,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:31,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:37:31,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:37:31,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:37:31,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:36,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:41,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:42,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:46,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:48,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:37:48,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:37:48,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:37:48,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:37:50,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:37:52,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:37:52,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:37:57,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:00,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:02,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:03,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:38:03,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:38:03,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:38:03,107 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:38:07,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:10,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:12,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:18,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:18,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:38:18,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:38:18,107 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:38:18,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:20,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:38:23,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:27,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:29,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:33,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:38:33,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:38:33,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:38:33,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:38:35,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:37,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:40,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:46,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:47,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:48,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:38:48,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:38:48,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:38:50,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:38:51,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:38:55,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:38:56,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:01,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:03,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:39:03,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:39:03,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:39:03,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:39:05,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:39:07,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:12,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:15,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:39:17,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:18,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:39:18,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:39:18,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:39:20,695 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:39:22,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:25,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:39:28,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:33,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:39:33,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:39:33,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:39:33,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:39:34,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:39,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:43,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:39:45,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:48,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:39:48,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:39:48,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:39:50,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:39:50,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:39:54,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:39:55,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:00,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:03,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:40:03,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:40:03,109 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:40:04,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:06,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:11,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:12,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:16,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:40:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:40:18,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:40:18,842 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:40:20,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:40:21,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:22,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:26,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:29,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:40:29,131 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:40:29,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:40:29,134 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:40:29,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:40:30,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:32,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:32,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:33,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:40:33,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:40:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:40:33,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:40:38,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:40,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:43,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:48,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:48,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:40:48,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:40:48,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:40:50,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:40:50,698 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:40:53,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:58,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:40:59,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:41:03,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:03,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:41:03,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:41:03,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:41:09,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:09,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:41:14,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:17,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:41:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:41:18,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:41:18,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:41:19,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:20,698 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:41:24,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:27,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:41:30,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:34,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:41:34,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:41:34,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:41:34,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:41:35,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:37,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:41:41,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:46,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:49,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:41:49,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:41:49,701 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:41:50,699 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:41:51,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:57,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:41:57,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:02,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:04,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:42:04,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:42:04,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:42:04,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:42:07,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:07,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:12,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:16,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:17,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:19,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:42:19,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:42:19,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:42:20,699 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:42:23,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:26,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:29,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:34,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:42:34,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:42:34,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:42:34,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:36,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:39,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:45,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:46,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:49,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:42:49,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:42:49,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:42:49,706 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:42:50,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:42:50,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:42:54,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:42:56,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:01,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:04,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:04,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:43:04,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:43:04,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:43:04,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:43:06,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:12,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:14,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:17,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:20,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:43:20,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:43:20,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:43:20,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:43:20,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:43:23,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:24,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:28,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:32,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:43:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:43:35,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:43:39,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:42,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:44,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:49,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:43:49,366 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:43:49,367 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:43:49,371 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:43:49,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:43:50,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:43:50,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:43:50,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:43:50,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:50,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:50,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:43:52,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:43:55,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:43:59,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:44:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:05,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:44:05,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:44:05,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:44:07,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:09,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:44:12,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:17,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:19,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:44:20,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:44:20,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:44:20,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:44:20,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:44:22,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:27,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:33,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:34,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:44:35,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:44:35,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:44:35,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:44:38,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:43,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:44,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:44:48,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:50,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:44:50,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:44:50,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:44:50,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:44:50,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:44:52,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:44:53,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:44:58,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:02,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:45:03,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:05,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:45:05,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:45:05,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:45:09,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:12,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:45:15,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:20,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:45:20,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:45:20,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:45:20,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:20,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:45:22,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:45:25,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:30,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:45:31,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:35,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:45:35,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:45:35,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:45:36,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:40,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:45:42,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:47,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:50,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:45:50,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:45:50,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:45:50,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:45:50,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:45:52,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:58,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:45:58,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:03,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:05,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:46:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:46:05,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:46:05,781 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:46:08,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:08,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:13,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:18,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:18,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:20,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:46:20,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:46:20,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:46:20,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:46:20,781 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:46:24,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:26,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:29,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:34,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:35,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:46:35,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:46:35,781 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:46:36,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:40,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:45,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:45,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:50,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:46:50,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:46:50,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:46:50,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:46:50,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:46:50,785 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:46:54,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:46:55,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:00,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:03,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:47:03,054 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:47:03,054 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:47:03,057 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:47:03,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:47:04,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:05,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:47:05,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:47:05,781 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:47:06,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:11,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:14,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:16,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:20,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:47:21,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:47:21,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:47:21,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:47:21,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:47:22,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:24,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:27,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:33,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:34,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:36,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:47:36,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:47:36,130 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:47:38,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:42,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:43,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:48,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:50,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:47:51,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:47:51,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:47:51,126 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:47:52,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:47:54,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:47:59,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:02,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:48:05,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:06,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:48:06,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:48:06,126 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:48:10,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:12,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:48:15,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:20,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:48:20,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:21,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:48:21,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:48:21,126 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:48:22,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:48:26,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:30,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:48:31,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:36,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:48:36,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:48:36,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:48:36,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:48:37,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:40,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:48:42,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:47,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:50,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:48:50,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:48:51,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:48:51,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:48:51,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:48:53,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:58,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:48:58,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:49:03,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:06,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:49:06,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:49:06,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:49:06,129 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:49:08,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:08,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:49:13,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:18,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:20,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:49:21,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:49:21,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:49:21,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:49:24,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:29,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:34,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:36,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:49:36,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:49:36,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:49:40,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:45,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:50,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:50,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:49:51,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:49:51,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:49:51,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:49:55,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:49:58,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:01,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:06,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:50:06,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:50:06,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:50:06,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:06,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:11,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:16,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:17,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:20,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:50:21,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:50:21,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:50:21,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:50:22,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:26,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:27,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:32,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:36,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:50:36,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:50:36,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:50:36,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:38,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:43,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:44,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:48,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:50,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:50:52,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:50:52,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:50:52,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:50:52,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:50:53,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:50:53,176 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:50:53,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:50:53,178 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:50:53,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:50:54,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:50:54,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:55,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:50:59,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:04,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:04,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:51:07,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:51:07,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:51:07,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:51:07,006 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:51:10,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:14,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:51:16,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:20,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:51:21,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:22,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:51:22,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:51:22,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:51:22,006 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:51:22,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:51:27,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:32,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:51:32,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:37,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:51:37,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:51:37,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:51:37,006 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:51:38,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:43,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:51:43,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:48,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:51:50,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:51:52,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:51:52,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:51:52,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:51:52,007 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:51:53,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:51:54,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:00,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:01,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:05,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:08,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:52:08,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:52:08,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:52:08,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:52:11,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:11,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:20,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:52:21,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:21,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:23,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:52:23,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:52:23,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:52:27,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:28,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:32,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:38,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:52:38,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:52:38,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:52:38,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:38,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:43,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:48,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:49,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:50,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:52:53,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:52:53,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:52:53,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:52:54,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:52:58,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:52:59,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:04,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:08,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:53:08,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:53:08,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:53:08,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:53:10,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:15,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:16,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:53:20,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:53:20,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:24,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:53:24,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:53:24,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:53:24,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:53:25,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:26,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:53:30,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:36,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:36,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:53:39,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:53:39,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:53:39,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:53:41,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:46,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:53:47,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:50,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:53:52,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:53:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:53:54,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:53:54,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:53:54,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:53:57,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:53:57,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:02,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:04,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:54:04,733 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:54:04,734 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:54:04,736 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:54:05,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:54:06,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:07,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:08,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:09,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:54:09,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:54:09,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:54:09,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:54:14,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:17,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:19,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:20,717 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:54:24,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:54:24,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:54:24,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:54:24,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:25,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:29,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:34,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:35,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:39,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:54:39,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:54:39,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:54:40,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:45,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:45,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:50,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:54:50,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:54:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:54:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:54:54,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:54:55,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:54:56,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:01,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:03,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:55:06,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:09,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:55:09,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:55:09,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:55:09,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:55:11,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:13,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:55:16,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:20,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:55:22,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:55:22,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:24,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:55:24,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:55:24,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:55:24,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:55:27,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:30,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:55:33,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:38,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:39,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:55:39,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:55:39,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:55:39,638 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:55:40,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:55:43,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:48,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:55:50,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:55:50,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:55:54,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:55:54,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:55:54,637 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:55:54,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:00,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:56:05,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:09,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:56:09,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:56:09,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:56:09,637 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:56:10,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:15,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:19,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:56:20,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:56:21,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:25,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:56:25,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:56:25,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:56:25,750 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:56:27,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:29,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:56:32,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:38,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:39,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:56:40,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:56:40,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:56:40,747 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:56:43,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:49,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:56:49,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:50,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:56:54,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:56:55,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:56:55,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:56:55,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:56:55,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:56:57,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:00,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:05,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:07,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:10,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:10,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:57:10,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:57:10,747 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:57:14,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-19 23:57:14,858 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-19 23:57:14,858 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-19 23:57:14,860 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-19 23:57:15,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-19 23:57:15,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:15,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:17,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:20,721 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:57:21,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:25,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:57:25,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:57:25,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:57:26,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:27,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:31,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:35,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:37,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:40,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:57:40,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:57:40,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:57:40,751 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:57:42,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:45,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:48,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:50,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:57:53,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:57:54,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:57:55,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:57:55,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:57:55,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:57:58,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:02,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:58:04,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:09,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:10,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:58:10,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:58:10,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:58:10,752 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:58:12,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:58:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:19,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:20,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:58:23,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:58:25,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:25,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:58:25,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:58:25,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:58:30,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:33,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:58:35,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:40,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:58:40,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:58:40,749 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:58:40,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:41,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:58:45,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:50,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:58:51,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:58:51,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:58:55,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:58:55,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:58:55,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:58:55,752 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:58:57,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:01,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:59:02,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:07,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:10,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:59:10,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:59:10,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:59:10,754 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:59:13,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:15,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:59:18,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:20,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:59:24,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:25,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:59:25,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:59:25,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:59:25,750 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:59:29,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:35,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:59:35,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:40,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:40,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-19 23:59:40,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:59:40,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:59:40,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:59:43,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:59:45,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:50,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-19 23:59:51,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-19 23:59:53,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-19 23:59:55,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-19 23:59:55,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-19 23:59:55,842 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-19 23:59:57,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:02,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:03,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:07,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:10,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:00:10,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:00:10,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:00:12,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:13,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:17,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:20,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:00:23,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:23,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:25,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:00:25,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:00:25,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:00:29,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:31,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:00:31,306 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:00:31,307 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:00:31,309 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:00:31,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:00:33,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:34,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:34,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:39,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:40,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:00:40,844 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:00:40,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:00:43,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:45,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:50,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:00:50,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:00:53,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:00:55,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:00:55,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:00:55,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:00:55,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:01,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:03,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:01:06,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:10,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:01:10,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:01:10,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:01:11,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:01:12,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:17,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:20,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:01:21,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:01:22,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:25,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:01:25,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:01:25,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:01:28,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:31,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:01:33,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:38,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:40,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:01:40,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:01:40,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:01:41,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:01:44,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:49,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:50,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:01:51,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:01:55,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:01:56,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:01:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:01:56,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:01:56,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:01:59,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:01,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:06,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:09,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:11,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:11,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:02:11,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:02:11,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:02:11,898 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:02:16,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:19,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:20,730 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:02:21,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:26,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:02:26,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:02:26,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:02:26,898 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:02:27,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:27,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:32,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:37,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:37,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:41,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:02:41,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:02:41,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:02:41,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:02:43,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:47,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:48,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:50,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:02:54,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:02:55,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:02:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:02:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:02:56,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:02:56,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:03:00,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:05,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:06,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:10,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:12,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:03:12,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:03:12,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:03:12,031 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:03:16,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:16,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:20,731 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:03:21,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:23,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:27,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:03:27,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:03:27,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:03:27,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:03:27,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:32,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:33,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:37,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:40,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:03:40,984 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:03:40,985 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:03:40,989 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:03:41,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:03:42,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:03:42,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:03:42,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:03:42,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:03:42,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:43,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:43,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:48,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:50,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:03:51,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:03:53,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:03:57,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:03:57,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:03:57,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:03:59,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:01,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:04:05,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:10,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:11,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:04:12,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:04:12,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:04:12,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:04:16,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:20,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:04:21,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:04:21,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:26,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:27,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:04:27,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:04:27,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:04:27,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:04:29,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:04:31,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:36,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:39,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:04:42,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:42,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:04:42,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:04:42,686 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:04:47,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:49,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:04:50,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:04:52,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:57,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:04:57,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:04:57,687 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:04:57,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:04:59,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:02,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:08,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:08,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:12,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:05:12,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:05:12,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:05:12,687 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:05:13,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:18,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:19,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:20,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:05:25,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:27,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:05:27,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:05:27,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:05:27,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:05:28,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:30,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:36,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:36,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:41,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:43,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:05:43,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:05:43,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:05:43,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:05:46,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:47,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:50,736 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:05:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:56,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:05:57,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:05:58,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:05:58,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:05:58,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:06:02,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:03,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:13,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:06:13,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:06:13,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:06:13,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:13,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:18,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:20,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:06:23,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:23,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:28,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:06:28,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:06:28,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:06:29,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:33,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:35,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:40,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:43,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:06:43,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:06:43,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:06:43,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:45,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:50,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:50,738 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:06:50,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:06:50,867 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:06:50,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:06:50,869 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:06:51,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:06:51,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:52,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:06:55,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:06:58,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:06:58,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:06:58,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:06:58,932 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:07:00,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:01,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:07:05,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:11,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:11,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:07:13,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:07:13,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:07:13,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:07:13,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:07:17,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:19,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:07:20,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:07:22,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:27,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:28,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:07:28,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:07:28,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:07:28,930 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:07:30,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:07:33,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:38,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:40,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:07:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:07:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:07:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:07:43,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:07:44,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:49,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:50,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:07:50,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:07:54,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:07:58,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:07:58,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:07:58,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:07:58,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:08:00,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:00,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:05,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:10,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:10,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:15,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:15,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:08:15,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:08:15,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:08:15,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:08:18,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:20,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:20,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:08:25,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:28,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:30,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:08:30,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:08:30,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:08:30,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:08:31,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:37,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:38,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:42,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:45,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:08:45,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:08:45,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:08:45,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:08:46,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:47,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:50,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:08:52,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:08:55,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:08:58,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:00,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:09:00,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:09:00,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:09:00,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:09:03,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:05,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:09:08,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:13,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:09:13,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:15,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:09:15,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:09:15,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:09:15,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:09:19,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:20,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:09:23,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:09:24,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:29,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:31,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:09:31,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:09:31,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:09:31,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:09:33,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:09:34,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:39,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:45,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:46,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:09:46,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:09:46,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:09:46,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:09:46,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:09:50,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:09:50,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:09:54,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:09:55,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:00,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:01,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:10:01,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:10:01,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:10:01,720 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:10:02,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:10:02,506 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:10:02,507 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:10:02,508 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:10:03,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:10:04,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:05,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:06,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:11,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:14,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:16,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:10:16,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:10:16,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:10:16,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:20,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:10:22,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:24,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:27,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:31,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:10:31,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:10:31,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:10:32,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:34,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:37,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:42,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:44,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:46,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:10:46,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:10:46,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:10:47,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:50,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:10:52,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:10:53,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:10:58,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:01,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:11:01,716 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:11:01,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:11:03,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:04,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:11:08,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:11,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:11:14,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:16,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:11:16,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:11:16,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:11:19,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:20,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:11:21,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:11:25,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:31,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:31,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:11:31,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:11:31,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:11:31,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:11:36,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:41,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:11:42,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:46,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:11:46,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:11:46,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:11:47,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:50,746 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:11:52,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:11:53,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:11:58,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:01,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:12:01,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:12:01,718 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:12:01,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:12:02,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:12:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:08,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:12,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:12:14,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:18,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:12:18,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:12:18,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:12:18,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:12:20,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:12:20,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:20,746 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:12:25,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:30,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:12:31,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:33,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:12:33,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:12:33,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:12:33,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:12:36,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:40,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:12:41,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:46,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:48,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:12:48,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:12:48,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:12:48,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:12:50,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:12:50,747 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:12:51,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:12:56,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:00,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:13:01,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:03,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:13:03,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:13:03,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:13:07,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:12,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:16,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:13:17,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:18,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:13:18,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:13:18,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:13:20,748 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:13:22,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:23,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:13:23,477 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:13:23,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:13:23,479 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:13:23,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:13:24,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:13:28,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:33,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:13:33,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:13:33,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:13:33,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:13:34,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:34,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:13:39,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:44,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:44,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:13:48,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:13:48,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:13:48,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:13:50,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:13:50,750 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:13:54,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:13:55,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:00,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:03,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:14:03,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:14:03,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:14:04,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:14:06,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:11,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:14,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:14:16,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:18,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:14:18,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:14:18,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:14:20,750 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:14:22,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:14:22,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:27,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:32,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:14:32,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:33,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:14:33,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:14:33,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:14:33,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:14:38,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:42,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:14:44,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:48,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:14:48,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:14:48,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:14:48,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:14:49,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:14:50,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:14:52,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:14:54,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:00,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:15:00,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:03,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:15:03,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:15:03,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:15:03,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:15:06,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:10,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:15:11,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:16,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:18,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:15:18,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:15:18,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:15:18,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:15:20,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:15:20,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:15:22,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:28,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:15:28,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:33,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:33,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:15:33,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:15:33,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:15:33,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:15:38,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:15:38,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:43,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:48,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:15:48,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:15:48,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:15:48,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:15:48,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:15:49,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:50,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:15:54,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:15:58,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:00,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:04,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:16:04,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:16:04,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:16:04,570 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:16:05,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:06,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:10,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:16,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:16,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:19,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:16:19,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:16:19,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:16:20,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:16:21,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:26,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:26,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:31,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:34,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:16:34,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:16:34,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:16:34,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:16:35,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:16:35,397 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:16:35,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:16:35,399 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:16:36,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:16:37,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:37,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:42,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:47,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:48,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:49,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:16:49,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:16:49,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:16:50,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:16:53,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:16:56,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:16:59,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:04,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:17:04,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:04,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:17:04,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:17:04,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:17:09,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:14,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:17:15,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:19,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:17:19,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:17:19,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:17:20,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:20,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:17:24,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:17:25,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:30,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:34,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:17:34,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:17:34,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:17:34,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:17:35,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:40,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:42,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:17:45,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:50,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:17:50,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:17:50,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:17:50,172 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:17:50,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:17:51,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:17:52,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:17:56,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:02,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:02,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:18:05,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:18:05,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:18:05,171 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:18:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:18:07,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:12,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:18:12,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:17,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:20,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:18:20,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:18:20,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:18:20,173 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:18:20,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:18:22,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:18:23,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:28,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:32,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:18:34,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:35,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:18:35,171 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:18:35,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:18:39,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:41,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:18:44,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:49,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:18:50,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:18:50,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:18:50,171 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:18:50,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:18:53,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:18:55,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:01,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:19:01,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:19:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:19:05,172 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:19:07,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:11,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:19:12,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:17,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:20,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:19:20,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:19:20,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:19:20,172 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:19:20,756 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:19:21,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:19:22,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:28,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:29,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:19:33,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:35,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:19:35,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:19:35,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:19:35,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:19:39,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:39,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:19:44,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:49,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:50,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:19:50,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:19:50,730 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:19:50,975 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:19:54,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:19:59,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:04,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:05,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:20:05,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:20:05,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: internal_messages -2023-11-20 00:20:05,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:20:08,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:20:08,707 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:20:08,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:20:08,710 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:20:09,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:20:10,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:10,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:20:11,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:20:15,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:20,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:20:20,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:20:20,730 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:20:20,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:20:20,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:25,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:30,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:20:31,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:36,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:36,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:20:36,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:20:36,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:20:36,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:20:38,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:20:41,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:46,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:48,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:20:50,758 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:20:51,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:20:51,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:20:51,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:20:52,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:57,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:20:58,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:02,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:06,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:21:06,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:21:06,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:21:08,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:09,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:13,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:18,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:19,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:20,758 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:21:21,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:21:21,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:21:21,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:21:24,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:27,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:29,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:34,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:36,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:21:36,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:21:36,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:21:36,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:21:37,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:40,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:46,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:47,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:50,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:21:51,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:21:53,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:21:53,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:21:53,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:21:53,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:21:57,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:21:57,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:02,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:05,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:22:07,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:08,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:22:08,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:22:08,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:22:08,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:22:12,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:15,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:22:17,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:20,760 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:22:23,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:22:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:22:23,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:22:24,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:22:28,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:32,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:22:33,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:38,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:22:38,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:22:38,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:22:38,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:22:39,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:42,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:22:45,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:50,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:22:50,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:22:52,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:22:53,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:22:53,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:22:53,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:22:55,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:00,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:00,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:05,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:08,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:08,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:08,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:23:08,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:23:10,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:11,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:16,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:18,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:23:18,639 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:23:18,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:23:18,642 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:23:18,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:23:19,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:20,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:23:20,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:21,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:23,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:23:23,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:23:27,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:29,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:33,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:38,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:38,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:38,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:38,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:23:38,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:23:39,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:43,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:49,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:23:49,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:50,762 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:23:54,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:54,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:23:54,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:23:54,681 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:23:54,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:23:57,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:00,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:05,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:07,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:09,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:24:09,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:24:09,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:24:09,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:24:11,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:17,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:17,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:20,762 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:24:22,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:24,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:24:24,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:24:24,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:24:25,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:27,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:32,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:35,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:38,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:39,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:24:39,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:24:39,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:24:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:45,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:48,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:50,764 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:24:53,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:24:54,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:24:54,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:24:54,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:24:54,734 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:24:59,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:03,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:25:05,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:10,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:10,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:25:10,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:25:10,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:25:13,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:25:15,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:20,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:20,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:25:22,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:25:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:25:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:25:25,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:25:25,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:30,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:32,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:25:36,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:40,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:25:40,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:25:40,470 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:25:42,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:42,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:25:47,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:50,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:25:51,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:25:52,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:25:55,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:25:55,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:25:55,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:25:58,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:01,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:04,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:09,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:10,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:26:10,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:26:10,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:26:11,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:15,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:19,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:20,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:26:20,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:25,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:26,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:26:26,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:26:26,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:26:26,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:26:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:26:27,807 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:26:27,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:26:27,810 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:26:28,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:26:29,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:30,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:30,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:35,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:39,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:41,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:41,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:26:41,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:26:41,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:26:46,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:49,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:26:50,769 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:26:51,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:56,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:26:56,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:26:56,672 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:26:56,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:26:57,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:27:01,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:07,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:27:07,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:11,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:27:11,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:27:11,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:27:11,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:27:12,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:17,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:27:18,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:20,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:27:23,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:25,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:27:26,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:27:26,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:27:26,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:27:26,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:27:28,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:33,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:35,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:27:39,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:41,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:27:41,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:27:41,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:27:44,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:49,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:27:50,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:27:50,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:55,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:27:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:27:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:27:56,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:27:59,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:01,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:06,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:10,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:11,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:11,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:28:11,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:28:11,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:28:16,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:20,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:20,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:28:22,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:26,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:28:26,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:28:26,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:28:27,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:27,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:32,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:37,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:38,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:41,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:28:41,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:28:41,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:28:43,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:47,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:49,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:50,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:28:54,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:28:55,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:28:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:28:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:28:56,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:28:59,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:05,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:05,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:29:10,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:12,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:29:12,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:29:12,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:29:12,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:29:15,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:15,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:29:20,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:20,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:29:23,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:29:25,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:27,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:29:27,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:29:27,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:29:30,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:33,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:29:35,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:40,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:42,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:29:42,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:29:42,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:29:42,269 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:29:43,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:29:43,420 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:29:43,420 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:29:43,423 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:29:43,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:29:45,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:29:46,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:50,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:29:51,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:29:55,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:29:57,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:29:57,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:29:57,270 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:29:57,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:02,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:05,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:30:08,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:30:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:30:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:30:12,533 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:30:13,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:15,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:30:18,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:20,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:30:24,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:26,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:30:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:30:27,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:30:27,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:30:29,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:34,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:30:34,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:39,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:30:42,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:30:42,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:30:44,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:30:45,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:50,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:30:51,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:54,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:30:57,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:30:57,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:30:57,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:30:57,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:31:02,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:03,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:07,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:31:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:31:12,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:31:12,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:13,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:17,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:20,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:31:23,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:23,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:27,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:31:27,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:31:27,536 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:31:28,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:31,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:34,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:39,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:41,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:42,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:31:42,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:31:42,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:31:44,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:47,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:48,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:50,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:50,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:31:54,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:55,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:31:56,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:31:57,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:31:57,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:31:57,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:32:01,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:03,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:32:06,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:11,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:32:12,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:32:12,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:32:13,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:32:16,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:20,776 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:32:22,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:23,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:32:27,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:27,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:32:27,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:32:27,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:32:33,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:33,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:32:38,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:41,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:32:42,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:32:42,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:32:42,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:32:43,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:48,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:50,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:32:50,769 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:32:50,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:32:50,772 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:32:50,777 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:32:51,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:32:52,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:32:53,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:32:57,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:32:57,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:32:57,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:32:59,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:02,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:33:05,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:11,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:12,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:33:12,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:33:12,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:33:12,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:33:16,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:20,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:33:22,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:33:22,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:27,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:28,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:33:28,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:33:28,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:33:32,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:33:33,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:38,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:40,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:33:43,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:33:43,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:33:44,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:49,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:33:50,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:50,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:33:55,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:33:58,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:33:58,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:33:59,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:01,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:06,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:07,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:12,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:13,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:34:13,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:34:17,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:17,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:20,779 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:34:23,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:27,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:28,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:34:28,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:34:29,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:34,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:38,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:40,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:44,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:34:44,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:34:44,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:34:44,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:34:45,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:48,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:50,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:50,779 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:34:55,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:34:56,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:34:59,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:34:59,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:34:59,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:34:59,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:35:01,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:06,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:35:06,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:11,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:14,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:35:14,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:35:16,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:35:17,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:20,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:35:22,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:24,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:35:28,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:29,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:35:29,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:35:29,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:35:33,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:34,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:35:38,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:44,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:44,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:35:44,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:35:44,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:35:49,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:35:50,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:35:54,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:35:55,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:00,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:00,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:36:00,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:36:00,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:36:01,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:36:01,479 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:36:01,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:36:01,482 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:36:01,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:36:02,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:36:05,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:10,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:15,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:36:15,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:36:15,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:36:15,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:20,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:36:20,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:21,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:36:25,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:30,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:36:30,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:36:30,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:36:31,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:31,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:36:36,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:41,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:36:42,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:45,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:36:45,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:36:45,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:36:47,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:50,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:36:51,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:36:52,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:36:57,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:00,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:37:00,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:37:00,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:37:01,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:37:03,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:08,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:10,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:37:14,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:15,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:37:15,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:37:15,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:37:19,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:20,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:37:20,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:37:24,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:29,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:30,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:37:30,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:37:30,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:37:30,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:37:35,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:40,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:37:40,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:45,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:46,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:37:46,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:37:46,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:37:48,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:37:50,783 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:37:51,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:56,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:37:58,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:38:01,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:38:01,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:38:01,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:38:02,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:08,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:38:08,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:13,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:16,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:38:16,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:38:16,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:38:18,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:38:18,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:20,784 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:38:23,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:26,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:38:29,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:31,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:38:31,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:38:31,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:38:34,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:36,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:38:40,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:45,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:46,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:38:46,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:38:46,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:38:46,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:38:50,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:50,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:38:55,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:38:56,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:00,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:02,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:39:02,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:39:02,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:39:02,091 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:39:03,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:06,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:11,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:13,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:16,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:17,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:39:17,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:39:17,091 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:39:20,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:39:21,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:22,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:39:22,368 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:39:22,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:39:22,373 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:39:23,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:39:24,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:25,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:27,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:32,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:39:32,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:39:32,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:39:32,239 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:39:32,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:34,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:38,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:44,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:44,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:47,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:39:47,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:39:47,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:39:49,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:50,788 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:39:54,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:39:54,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:39:59,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:02,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:40:02,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:40:02,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:40:04,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:40:05,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:10,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:12,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:40:15,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:17,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:40:17,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:40:17,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:40:20,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:40:20,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:22,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:40:26,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:32,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:32,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:40:32,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:40:32,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:40:32,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:40:37,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:42,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:40:42,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:47,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:40:47,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:40:47,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:40:47,242 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:40:48,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:50,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:40:52,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:40:53,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:40:58,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:00,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:02,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:41:02,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:41:04,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:10,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:10,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:15,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:18,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:41:18,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:41:18,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:41:18,063 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:41:20,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:41:20,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:20,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:25,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:31,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:31,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:33,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:41:33,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:41:33,062 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:41:37,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:40,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:43,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:48,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:41:48,066 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:41:48,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:48,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:50,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:41:53,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:41:58,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:41:59,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:03,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:42:03,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:42:03,067 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:42:03,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:42:05,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:10,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:15,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:18,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:42:18,067 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:42:20,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:42:20,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:25,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:30,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:33,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:42:33,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:42:33,094 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:42:36,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:36,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:42:41,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:44,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:42:47,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:48,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:42:48,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:42:48,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:42:50,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:42:52,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:42:54,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:42:58,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:02,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:43:02,839 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:43:02,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:43:02,849 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:43:03,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:43:03,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:43:03,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:43:03,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:43:03,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:04,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:43:06,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:43:09,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:14,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:18,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:43:18,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:43:18,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:43:19,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:20,793 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:43:24,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:29,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:33,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:43:33,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:43:33,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:43:35,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:40,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:45,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:48,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:43:48,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:43:48,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:43:50,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:43:50,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:43:55,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:00,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:03,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:44:03,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:44:03,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:44:06,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:11,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:16,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:18,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:44:18,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:44:18,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:44:20,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:44:21,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:26,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:31,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:33,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:44:33,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:44:33,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:44:37,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:42,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:47,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:48,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:44:48,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:44:48,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:44:50,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:44:52,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:44:57,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:02,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:03,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:45:03,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:45:03,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:45:08,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:13,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:18,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:45:18,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:45:18,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:45:18,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:20,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:45:23,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:28,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:33,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:45:33,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:45:33,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:45:34,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:39,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:44,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:48,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:45:48,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:45:48,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:45:50,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:45:50,796 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:45:55,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:00,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:03,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:46:03,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:46:03,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:46:06,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:11,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:16,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:18,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:46:18,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:46:18,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:46:20,797 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:46:21,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:26,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:31,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:33,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:46:33,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:46:33,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:46:37,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:42,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:47,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:48,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:46:48,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:46:48,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:46:50,797 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:46:52,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:46:57,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:02,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:03,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:47:03,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:47:03,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:47:08,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:13,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:18,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:47:18,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:47:18,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:47:19,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:20,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:47:24,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:29,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:33,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:47:33,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:47:33,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:47:35,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:40,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:45,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:48,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:47:48,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:47:48,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:47:50,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:47:50,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:47:55,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:00,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:03,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:48:03,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:48:03,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:48:06,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:11,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:16,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:18,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:48:18,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:48:18,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:48:20,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:48:21,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:26,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:31,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:33,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:48:33,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:48:33,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:48:37,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:42,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:47,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:48,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:48:48,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:48:48,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:48:50,799 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:48:52,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:48:57,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:02,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:03,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:49:03,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:49:03,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:49:08,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:13,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:18,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:49:18,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:49:18,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:49:19,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:20,800 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:49:24,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:29,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:33,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:49:33,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:49:33,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:49:35,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:40,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:45,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:48,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:49:48,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:49:48,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:49:50,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:49:50,800 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:49:55,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:00,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:03,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:50:03,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:50:03,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:50:06,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:11,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:16,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:18,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:50:18,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:50:18,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:50:20,801 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:50:21,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:26,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:31,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:33,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:50:33,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:50:33,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:50:37,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:42,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:47,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:48,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:50:48,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:50:48,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:50:50,801 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:50:52,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:50:57,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:02,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:03,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:51:03,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:51:03,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:51:08,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:13,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:18,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:51:18,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:51:18,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:51:18,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:20,802 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:51:23,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:28,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:33,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:51:33,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:51:33,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:51:34,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:39,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:44,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:48,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:51:48,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:51:48,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:51:50,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:51:50,802 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:51:55,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:00,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:03,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:52:03,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:52:03,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:52:06,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:11,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:16,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:18,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:52:18,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:52:18,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:52:20,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:52:21,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:26,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:31,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:33,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:52:33,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:52:33,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:52:37,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:42,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:47,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:48,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:52:48,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:52:48,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:52:50,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:52:52,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:52:57,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:02,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:03,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:53:03,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:53:03,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:53:08,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:13,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:18,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:53:18,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:53:18,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:53:19,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:20,804 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:53:24,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:29,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:33,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:53:33,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:53:33,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:53:33,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:53:33,804 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:53:33,804 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:53:33,807 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:53:34,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:53:34,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:35,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:53:39,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:44,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:48,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:53:48,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:53:48,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:53:50,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:50,804 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:53:55,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:53:59,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:54:00,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:03,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:54:03,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:03,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:54:03,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:06,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:16,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:18,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:18,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:19,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:54:19,139 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:54:20,805 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:54:21,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:26,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:31,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:33,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:33,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:33,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:54:33,974 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:54:37,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:43,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:54:43,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:44,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:54:48,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:54:48,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:50,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:54:53,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:54:53,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:54:58,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:03,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:06,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:55:06,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:55:08,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:55:08,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:55:09,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:55:09,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:14,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:14,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:19,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:20,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:55:23,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:23,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:55:23,028 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:55:25,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:29,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:30,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:35,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:38,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:38,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:55:38,028 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:55:41,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:44,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:46,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:50,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:55:51,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:53,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:55:53,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:55:53,028 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:55:57,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:55:59,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:02,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:07,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:56:07,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:08,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:08,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:56:08,028 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:56:13,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:16,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:18,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:19,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:56:20,817 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:56:23,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:23,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:56:23,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:56:24,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:29,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:29,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:56:31,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:34,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:38,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:38,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:56:38,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:56:39,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:56:40,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:45,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:46,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:49,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:56:50,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:50,807 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:56:53,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:56:53,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:56:53,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:56:56,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:56:59,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:01,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:02,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:07,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:08,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:08,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:08,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:57:08,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:57:13,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:16,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:18,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:18,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:20,808 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:57:23,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:23,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:57:23,033 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:57:24,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:28,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:30,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:31,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:35,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:37,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:38,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:38,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:57:38,030 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:57:40,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:45,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:46,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:47,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:50,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:57:50,808 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:57:54,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:57:54,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:57:54,030 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:57:55,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:57:56,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:01,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:01,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:58:05,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:58:06,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:09,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:58:09,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:58:12,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:15,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:58:16,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:58:17,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:20,809 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:58:22,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:24,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:58:24,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:58:25,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:58:28,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:32,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:58:33,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:35,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:58:38,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:39,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:58:39,024 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:58:43,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:58:43,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:47,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:58:48,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:50,811 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:58:53,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:58:54,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:58:54,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:58:54,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:58:59,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:02,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:59:03,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:59:05,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:09,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:59:09,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:59:10,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:13,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:59:16,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:17,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:59:20,813 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:59:21,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:22,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 00:59:22,805 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 00:59:22,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 00:59:22,809 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 00:59:23,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 00:59:24,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:59:24,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:59:24,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:59:27,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:32,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:32,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:59:34,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:59:37,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:39,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:59:39,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:59:43,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:44,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:59:47,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 00:59:48,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:50,814 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 00:59:53,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 00:59:54,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 00:59:54,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 00:59:54,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 00:59:59,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:02,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:04,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:00:04,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:09,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:10,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:10,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:00:10,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:00:14,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:00:15,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:17,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:20,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:20,814 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:00:22,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:00:25,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:25,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:00:25,222 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:00:26,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:31,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:00:31,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:32,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:36,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:40,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:40,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:00:40,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:00:41,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:00:42,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:47,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:47,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:50,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:00:51,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:00:53,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:00:55,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:00:55,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:00:55,223 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:00:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:01,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:01:02,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:04,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:09,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:10,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:10,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:01:10,225 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:01:11,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:01:14,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:18,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:19,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:01:20,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:20,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:01:25,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:25,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:01:25,226 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:01:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:29,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:01:30,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:33,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:35,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:39,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:01:40,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:40,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:01:40,226 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:01:41,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:46,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:48,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:50,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:01:50,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:01:51,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:01:56,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:01:56,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:01:56,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:01:56,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:00,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:02,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:03,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:02:07,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:08,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:11,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:02:11,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:02:13,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:18,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:18,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:02:19,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:20,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:02:24,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:26,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:26,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:02:26,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:02:30,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:33,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:02:35,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:02:35,125 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:02:35,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:02:35,128 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:02:35,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:02:36,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:36,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:41,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:41,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:02:41,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:02:41,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:02:46,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:46,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:48,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:02:50,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:02:51,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:02:56,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:02:56,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:02:56,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:02:56,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:02:57,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:02,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:03,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:04,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:03:08,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:11,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:11,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:03:11,727 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:03:14,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:14,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:03:18,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:19,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:20,818 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:03:23,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:03:25,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:26,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:26,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:03:26,727 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:03:30,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:33,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:03:33,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:36,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:41,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:41,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:41,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:03:41,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:03:43,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:03:46,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:48,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:50,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:03:51,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:03:53,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:03:56,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:03:56,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:03:56,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:03:56,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:01,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:04:02,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:03,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:07,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:11,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:11,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:04:11,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:04:12,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:04:12,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:17,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:19,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:20,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:04:22,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:04:23,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:26,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:26,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:04:26,734 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:04:28,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:32,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:04:34,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:34,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:39,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:41,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:41,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:04:41,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:04:42,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:04:45,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:49,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:50,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:04:50,820 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:04:50,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:55,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:04:57,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:04:57,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:04:57,724 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:05:00,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:01,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:04,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:06,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:10,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:12,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:12,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:12,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:05:12,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:05:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:19,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:20,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:20,820 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:05:22,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:27,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:27,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:05:27,724 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:05:27,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:30,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:32,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:34,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:38,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:40,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:42,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:42,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:05:42,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:05:43,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:47,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:05:47,114 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:05:47,115 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:05:47,118 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:05:47,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:05:48,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:49,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:49,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:50,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:05:54,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:05:57,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:05:57,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:05:57,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:05:58,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:05:59,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:04,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:05,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:08,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:06:10,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:12,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:12,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:06:12,727 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:06:15,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:17,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:06:20,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:20,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:06:21,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:25,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:06:27,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:27,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:27,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:06:27,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:06:32,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:35,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:36,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:06:38,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:43,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:43,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:43,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:06:43,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:06:46,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:06:48,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:50,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:50,822 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:06:53,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:06:56,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:06:58,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:06:58,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:06:58,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:06:58,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:03,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:05,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:06,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:07:09,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:13,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:13,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:07:13,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:07:14,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:16,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:07:19,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:20,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:20,824 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:07:24,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:07:25,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:28,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:07:28,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:28,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:07:31,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:34,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:07:35,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:37,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:42,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:43,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:07:43,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:43,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:07:44,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:07:48,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:50,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:50,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:07:53,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:07:54,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:07:58,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:07:58,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:07:58,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:07:59,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:04,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:08:05,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:05,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:08:10,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:12,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:08:13,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:08:13,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:08:15,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:20,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:08:20,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:08:21,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:22,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:08:26,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:28,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:08:28,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:08:28,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:08:32,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:32,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:08:35,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:08:37,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:40,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:08:43,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:43,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:08:43,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:08:48,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:50,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:08:50,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:08:52,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:08:54,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:08:58,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:08:58,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:08:59,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:08:59,621 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:08:59,622 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:08:59,624 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:09:00,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:09:00,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:01,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:03,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:05,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:05,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:09:10,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:11,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:13,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:09:13,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:09:16,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:20,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:20,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:09:20,826 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:09:22,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:27,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:28,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:09:28,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:09:30,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:32,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:35,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:09:37,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:42,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:43,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:09:43,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:09:44,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:47,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:50,828 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:09:51,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:09:53,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:54,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:09:58,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:09:58,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:09:58,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:10:03,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:04,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:10:06,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:08,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:13,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:10:13,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:10:13,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:14,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:10:18,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:20,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:10:21,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:22,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:10:24,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:29,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:29,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:29,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:10:29,713 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:10:32,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:10:34,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:36,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:39,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:42,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:10:44,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:44,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:10:44,712 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:10:44,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:49,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:50,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:10:51,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:52,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:10:55,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:10:59,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:10:59,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:10:59,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:11:01,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:02,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:06,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:06,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:11,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:12,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:14,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:14,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:11:14,714 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:11:17,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:20,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:11:21,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:21,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:23,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:29,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:29,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:29,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:11:29,714 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:11:31,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:34,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:37,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:40,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:41,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:44,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:44,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:11:44,714 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:11:45,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:50,830 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:11:51,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:51,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:52,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:56,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:11:59,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:11:59,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:11:59,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:11:59,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:12:02,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:07,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:07,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:08,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:12:12,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:15,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:15,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:12:15,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:12:16,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:12:16,403 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:12:16,403 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:12:16,406 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:12:16,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:12:18,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:18,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:12:19,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:12:20,830 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:12:22,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:23,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:28,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:12:29,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:30,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:12:30,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:30,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:12:34,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:36,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:12:37,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:39,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:44,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:45,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:12:45,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:45,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:12:46,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:12:50,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:50,831 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:12:53,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:12:55,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:12:56,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:00,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:13:00,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:13:01,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:06,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:07,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:08,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:13:12,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:14,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:15,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:13:15,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:13:17,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:20,833 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:13:22,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:23,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:13:26,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:28,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:30,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:13:30,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:13:33,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:36,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:38,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:13:38,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:43,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:45,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:45,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:13:45,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:13:49,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:13:50,834 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:13:53,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:13:55,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:13:55,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:00,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:00,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:14:00,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:14:00,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:05,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:14:05,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:08,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:10,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:15,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:14:15,230 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:14:15,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:15,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:14:16,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:20,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:14:21,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:23,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:25,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:14:27,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:30,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:14:30,418 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:14:32,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:33,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:14:37,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:39,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:42,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:43,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:14:45,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:45,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:14:45,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:14:48,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:50,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:14:52,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:14:53,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:14:54,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:14:58,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:00,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:00,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:15:00,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:15:00,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:15:04,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:09,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:09,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:10,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:15:14,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:15,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:15,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:15:15,416 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:15:20,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:20,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:15:20,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:15:24,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:25,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:28,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:15:28,880 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:15:28,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:15:28,884 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:15:29,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:15:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:15:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:30,417 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:15:30,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:15:31,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:36,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:39,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:40,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:15:42,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:46,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:46,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:15:46,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:15:47,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:48,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:15:50,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:15:52,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:56,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:15:58,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:15:58,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:16:01,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:01,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:16:01,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:16:03,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:08,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:09,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:16:11,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:13,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:16,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:16,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:16:16,526 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:16:17,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:16:19,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:20,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:16:24,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:26,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:27,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:16:30,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:31,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:31,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:16:31,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:16:36,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:37,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:16:41,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:41,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:46,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:46,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:16:46,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:16:46,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:47,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:16:50,837 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:16:51,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:56,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:16:57,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:16:57,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:01,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:17:01,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:17:01,529 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:17:02,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:07,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:08,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:11,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:17:13,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:15,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:16,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:17:16,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:17:16,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:17:18,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:20,839 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:17:24,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:25,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:26,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:17:29,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:31,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:17:31,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:17:31,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:17:34,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:35,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:39,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:41,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:17:45,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:45,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:46,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:17:46,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:17:50,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:17:50,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:52,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:17:56,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:17:56,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:01,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:01,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:18:01,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:01,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:18:02,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:18:06,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:11,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:11,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:12,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:18:16,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:16,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:18:16,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:18:20,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:18:21,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:22,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:18:26,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:26,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:31,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:18:31,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:18:31,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:18:32,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:37,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:39,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:18:39,771 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:18:39,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:18:39,775 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:18:40,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:18:41,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:18:41,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:42,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:47,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:47,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:47,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:18:47,932 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:18:50,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:18:51,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:18:52,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:18:57,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:18:57,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:01,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:02,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:02,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:02,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:19:02,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:19:08,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:09,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:12,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:13,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:17,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:17,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:19:17,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:19:19,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:19,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:20,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:19:24,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:27,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:29,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:29,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:32,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:32,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:19:32,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:19:35,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:37,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:41,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:42,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:46,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:47,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:47,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:19:47,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:19:47,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:19:50,841 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:19:51,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:57,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:19:57,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:19:57,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:02,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:04,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:20:04,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:20:07,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:07,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:20:12,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:13,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:17,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:17,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:20:19,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:19,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:20:19,385 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:20:20,842 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:20:22,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:25,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:20:28,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:28,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:33,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:34,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:34,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:20:34,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:20:35,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:20:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:43,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:45,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:20:48,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:49,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:49,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:20:49,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:20:50,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:20:54,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:20:55,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:20:58,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:20:59,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:21:04,386 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:21:05,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:05,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:10,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:13,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:13,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:16,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:20,845 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:21:20,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:20,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:21:20,868 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:21:22,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:23,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:27,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:28,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:32,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:33,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:35,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:35,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:21:35,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:21:38,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:43,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:43,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:43,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:48,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:50,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:21:50,777 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:21:50,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:21:50,780 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:21:50,845 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:21:50,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:50,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:21:50,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:21:51,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:21:51,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:53,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:21:54,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:21:59,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:21:59,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:01,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:22:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:05,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:22:05,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:22:05,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:22:09,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:11,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:22:14,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:22:14,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:20,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:20,845 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:22:20,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:22:20,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:22:20,867 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:22:21,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:22:26,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:29,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:22:29,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:22:31,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:36,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:37,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:22:37,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:22:37,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:22:39,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:22:42,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:44,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:22:47,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:49,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:22:50,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:22:52,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:22:52,242 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:22:52,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:58,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:22:59,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:23:00,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:23:03,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:07,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:23:07,242 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:23:08,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:23:08,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:13,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:15,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:23:17,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:23:19,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:20,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:23:22,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:23:22,242 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:23:24,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:27,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:23:29,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:30,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:23:34,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:37,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:23:37,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:23:37,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:23:40,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:45,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:23:45,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:50,847 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:23:50,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:51,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:23:52,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:23:52,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:23:56,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:23:59,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:24:00,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:24:01,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:06,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:07,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:24:07,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:24:09,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:24:12,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:16,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:24:17,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:19,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:24:20,847 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:24:22,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:24:22,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:24:22,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:27,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:24:27,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:31,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:24:32,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:37,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:24:37,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:24:37,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:24:37,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:24:38,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:43,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:46,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:24:47,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:24:48,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:50,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:24:52,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:24:52,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:24:54,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:24:57,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:00,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:01,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:04,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:25:04,380 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:25:04,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:25:04,384 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:25:04,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:25:05,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:05,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:07,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:25:07,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:25:10,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:16,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:16,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:16,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:20,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:25:21,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:22,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:25:22,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:25:26,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:27,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:31,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:32,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:34,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:25:37,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:37,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:25:37,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:42,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:44,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:46,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:47,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:50,850 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:25:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:25:52,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:25:52,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:25:53,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:25:54,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:25:58,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:01,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:26:01,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:03,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:07,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:26:07,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:26:09,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:11,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:26:14,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:16,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:19,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:20,850 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:26:21,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:26:22,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:26:22,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:22,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:26:25,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:31,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:31,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:26:31,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:36,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:37,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:37,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:26:37,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:26:39,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:26:41,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:46,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:47,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:49,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:26:50,851 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:26:51,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:52,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:26:52,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:26:52,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:26:57,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:26:59,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:02,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:02,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:07,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:07,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:27:07,601 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:27:08,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:09,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:13,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:17,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:19,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:19,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:20,851 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:27:22,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:22,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:27:22,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:27:24,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:27,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:30,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:32,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:35,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:37,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:37,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:27:37,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:27:38,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:40,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:46,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:47,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:48,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:50,852 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:27:51,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:27:54,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:27:54,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:27:54,024 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:27:56,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:27:57,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:02,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:03,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:06,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:28:07,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:09,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:09,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:28:09,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:28:13,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:13,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:28:13,892 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:28:13,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:28:13,894 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:28:14,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:28:16,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:28:18,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:18,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:20,853 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:28:24,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:24,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:28:24,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:28:24,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:24,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:28:29,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:33,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:34,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:28:34,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:39,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:39,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:28:39,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:28:40,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:44,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:28:45,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:48,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:50,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:28:50,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:28:53,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:28:54,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:28:54,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:28:54,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:28:56,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:01,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:03,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:03,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:29:06,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:09,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:09,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:29:09,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:29:11,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:11,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:29:16,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:19,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:20,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:29:21,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:29:21,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:24,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:24,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:29:24,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:29:26,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:31,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:29:32,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:34,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:37,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:39,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:39,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:29:39,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:29:41,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:29:42,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:47,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:49,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:50,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:29:50,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:29:53,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:29:54,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:29:54,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:29:54,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:29:59,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:00,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:04,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:04,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:30:09,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:30:09,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:30:09,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:30:09,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:10,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:14,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:19,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:30:20,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:20,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:20,856 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:30:24,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:30:24,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:30:24,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:30:25,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:30,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:31,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:35,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:30:36,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:38,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:39,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:30:39,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:30:41,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:47,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:48,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:50,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:30:50,857 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:30:52,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:30:54,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:30:54,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:30:58,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:30:58,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:03,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:05,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:31:06,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:31:09,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:09,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:31:09,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:31:14,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:16,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:31:20,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:20,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:31:20,857 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:31:24,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:31:24,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:31:24,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:31:24,875 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:31:24,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:31:24,877 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:31:25,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:31:25,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:26,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:31:30,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:35,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:31:36,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:36,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:31:39,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:31:39,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:31:41,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:45,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:31:47,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:50,864 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:31:51,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:31:52,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:31:53,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:31:54,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:31:54,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:31:58,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:03,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:32:04,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:06,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:32:09,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:11,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:32:11,128 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:32:14,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:32:15,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:20,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:20,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:32:21,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:32:22,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:32:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:26,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:32:26,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:32:30,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:32,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:32:35,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:36,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:32:40,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:41,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:32:41,128 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:32:42,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:32:46,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:50,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:32:50,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:32:51,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:51,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:32:56,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:32:57,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:32:57,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:32:57,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:33:00,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:33:02,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:07,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:07,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:33:10,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:33:12,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:33:12,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:33:12,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:17,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:20,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:33:20,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:33:22,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:33:22,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:27,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:33:27,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:33:28,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:30,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:33:33,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:37,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:33:38,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:33:39,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:42,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:33:42,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:33:44,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:48,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:33:50,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:50,861 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:33:52,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:33:55,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:33:57,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:33:57,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:33:58,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:01,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:07,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:34:07,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:08,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:12,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:14,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:34:14,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:34:14,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:34:16,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:18,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:20,861 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:34:23,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:34:23,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:26,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:29,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:34:29,155 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:34:29,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:33,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:34:33,713 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:34:33,714 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:34:33,716 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:34:33,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:34:33,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:34,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:34,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:38,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:34:39,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:44,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:44,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:34:44,156 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:34:45,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:50,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:50,863 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:34:53,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:34:54,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:34:55,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:34:59,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:34:59,156 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:35:01,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:06,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:08,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:35:11,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:14,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:35:14,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:35:16,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:18,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:35:20,863 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:35:22,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:24,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:35:26,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:35:28,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:29,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:35:29,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:35:33,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:36,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:35:38,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:39,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:35:43,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:44,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:35:44,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:35:46,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:35:49,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:50,864 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:35:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:35:55,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:35:56,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:35:59,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:35:59,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:36:00,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:06,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:06,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:36:09,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:36:11,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:14,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:36:14,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:36:16,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:16,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:36:20,865 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:36:21,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:24,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:36:24,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:36:27,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:29,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:36:29,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:36:32,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:34,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:36:37,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:39,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:36:43,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:44,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:36:44,157 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:36:44,844 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:36:48,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:50,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:36:52,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:36:53,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:36:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:36:58,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:00,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:37:00,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:37:00,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:37:02,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:37:04,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:09,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:09,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:37:13,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:37:14,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:15,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:37:15,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:37:19,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:20,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:37:21,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:37:24,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:37:24,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:30,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:37:30,089 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:37:30,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:31,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:37:35,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:39,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:37:40,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:37:41,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:45,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:37:45,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:37:46,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:50,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:37:50,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:37:51,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:54,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:37:56,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:37:57,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:37:57,769 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:37:57,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:37:57,772 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:37:58,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:37:58,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:38:00,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:38:00,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:38:02,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:07,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:10,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:38:12,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:38:12,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:15,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:38:15,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:38:18,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:20,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:38:22,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:38:23,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:25,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:38:28,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:30,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:38:30,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:38:32,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:38:33,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:38,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:40,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:38:40,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:38:43,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:45,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:38:45,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:38:49,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:50,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:38:50,869 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:38:54,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:38:55,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:39:00,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:39:00,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:39:00,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:00,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:39:05,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:08,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:39:10,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:39:11,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:15,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:39:15,078 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:39:16,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:18,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:39:20,869 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:39:21,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:25,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:39:26,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:29,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:39:30,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:39:30,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:39:32,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:37,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:40,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:39:41,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:39:43,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:45,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:39:45,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:39:49,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:49,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:39:50,870 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:39:54,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:39:56,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:39:59,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:00,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:40:00,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:40:00,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:05,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:09,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:10,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:11,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:40:15,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:15,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:40:15,947 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:40:19,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:20,870 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:40:20,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:25,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:26,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:40:28,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:30,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:40:30,945 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:40:31,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:36,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:36,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:41,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:40:41,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:45,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:40:45,945 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:40:46,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:47,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:50,871 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:40:52,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:40:56,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:40:56,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:40:58,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:00,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:41:00,945 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:41:04,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:06,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:41:09,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:11,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:41:14,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:41:14,631 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:41:14,631 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:41:14,635 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:41:14,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:41:15,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:15,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:41:15,946 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:41:16,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:41:20,871 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:41:20,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:24,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:41:26,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:26,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:41:31,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:32,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:41:32,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:41:32,069 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:41:34,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:41:36,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:41,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:41,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:41:45,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:41:46,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:47,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:41:47,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:41:47,068 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:41:50,874 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:41:51,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:41:53,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:41:56,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:41:56,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:02,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:02,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:42:02,068 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:42:02,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:03,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:42:07,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:11,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:13,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:13,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:42:17,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:17,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:42:17,069 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:42:18,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:20,873 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:42:23,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:42:23,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:26,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:28,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:31,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:42:32,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:32,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:42:32,068 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:42:34,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:39,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:41,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:42:41,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:44,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:48,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:42:48,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:42:48,222 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:42:50,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:50,874 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:42:51,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:42:55,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:42:57,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:01,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:01,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:43:03,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:03,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:43:03,222 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:43:06,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:10,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:43:11,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:12,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:16,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:18,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:18,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:43:18,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:43:20,874 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:43:21,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:22,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:43:26,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:27,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:31,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:33,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:33,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:43:33,230 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:43:37,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:40,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:43:42,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:42,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:47,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:48,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:48,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:43:48,265 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:43:50,875 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:43:50,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:43:52,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:43:57,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:43:57,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:00,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:03,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:44:03,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:44:03,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:09,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:09,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:12,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:44:14,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:18,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:44:18,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:44:19,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:19,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:20,876 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:44:24,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:27,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:44:29,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:33,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:44:33,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:44:35,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:36,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:44:36,554 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:44:36,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:44:36,558 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:44:37,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:44:39,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:40,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:44:45,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:47,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:48,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:44:48,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:44:50,876 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:44:50,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:55,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:44:57,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:44:57,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:45:01,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:04,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:45:04,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:45:04,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:45:06,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:07,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:45:11,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:45:15,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:45:16,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:19,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:45:19,258 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:45:20,888 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:45:21,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:25,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:45:26,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:28,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:45:31,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:34,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:45:34,258 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:45:35,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:45:37,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:42,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:45:45,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:45:47,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:49,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:45:49,259 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:45:50,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:45:52,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:55,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:45:57,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:45:58,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:46:03,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:03,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:46:04,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:46:04,258 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:46:09,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:12,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:46:13,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:46:14,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:19,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:46:20,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:46:20,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:46:20,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:46:22,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:46:25,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:29,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:46:30,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:30,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:46:35,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:46:35,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:46:35,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:46:35,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:41,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:46:41,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:44,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:46:46,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:50,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:46:50,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:46:50,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:46:51,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:46:51,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:56,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:46:59,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:01,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:01,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:05,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:47:05,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:47:07,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:09,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:13,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:14,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:18,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:19,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:47:20,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:20,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:47:20,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:47:23,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:28,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:29,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:29,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:33,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:35,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:35,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:47:35,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:47:39,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:39,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:44,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:44,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:46,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:47:46,085 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:47:46,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:47:46,088 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:47:46,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:47:47,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:50,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:50,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:47:50,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:47:50,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:47:50,949 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:47:55,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:47:57,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:47:59,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:48:00,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:05,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:48:05,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:48:05,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:48:06,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:07,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:48:11,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:14,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:48:16,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:17,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:48:20,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:48:20,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:48:21,039 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:48:22,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:27,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:27,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:48:29,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:48:32,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:35,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:48:35,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:48:37,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:48:37,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:42,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:44,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:48:45,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:48:48,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:50,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:48:50,780 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:48:50,993 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:48:53,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:56,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:48:58,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:48:59,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:49:03,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:05,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:49:05,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:49:07,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:49:09,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:14,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:15,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:49:17,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:49:19,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:20,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:49:20,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:49:20,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:49:24,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:27,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:49:30,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:49:30,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:35,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:37,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:49:37,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:49:37,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:49:39,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:49:41,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:45,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:49:46,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:49,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:49:50,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:49:51,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:52,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:49:52,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:49:52,166 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:49:57,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:49:59,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:50:00,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:03,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:07,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:07,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:50:07,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:50:08,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:09,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:50:13,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:15,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:19,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:19,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:50:20,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:50:22,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:22,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:50:22,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:50:24,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:29,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:50:30,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:30,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:35,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:37,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:37,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:50:37,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:50:39,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:50:40,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:45,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:45,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:49,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:50:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:50,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:50:52,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:50:52,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:50:52,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:50:55,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:50:59,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:51:00,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:51:01,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:06,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:07,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:51:07,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:51:08,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:51:08,183 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:51:08,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:51:08,186 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:51:08,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:51:09,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:51:12,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:15,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:51:17,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:20,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:51:20,886 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:51:22,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:51:22,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:51:22,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:28,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:30,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:51:30,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:51:33,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:37,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:51:37,414 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:51:39,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:40,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:51:44,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:46,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:51:50,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:51:50,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:50,886 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:51:52,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:51:52,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:51:55,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:51:57,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:52:00,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:01,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:05,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:07,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:52:07,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:07,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:52:07,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:52:11,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:16,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:17,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:52:17,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:20,887 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:52:22,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:22,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:52:22,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:52:22,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:27,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:52:28,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:31,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:33,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:37,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:37,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:52:37,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:52:37,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:52:38,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:43,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:46,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:48,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:50,887 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:52:51,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:52:52,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:52:52,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:52:52,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:52:54,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:52:59,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:01,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:01,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:53:05,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:07,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:07,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:53:07,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:53:11,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:11,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:53:16,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:16,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:20,888 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:53:21,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:21,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:53:22,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:22,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:53:22,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:53:27,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:31,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:31,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:53:33,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:38,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:38,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:53:38,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:53:39,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:42,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:53:44,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:47,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:53:49,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:53:50,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:53:50,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:53:53,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:53:53,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:53:55,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:00,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:01,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:02,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:54:06,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:08,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:54:08,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:54:10,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:11,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:16,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:17,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:54:20,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:20,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:54:21,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:23,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:54:23,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:54:26,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:54:26,830 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:54:26,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:54:26,834 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:54:27,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:54:27,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:27,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:28,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:32,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:54:32,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:37,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:38,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:54:38,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:54:38,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:43,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:47,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:54:48,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:54:48,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:50,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:54:53,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:54:53,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:54:54,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:54:57,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:00,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:02,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:55:05,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:07,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:08,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:55:08,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:55:10,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:15,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:17,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:17,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:55:20,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:20,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:55:23,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:55:23,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:55:23,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:55:26,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:27,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:31,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:32,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:55:36,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:37,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:38,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:55:38,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:55:42,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:45,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:47,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:55:48,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:50,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:55:53,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:55:54,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:55:54,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:55:54,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:55:55,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:55:59,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:03,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:56:04,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:06,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:56:09,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:56:09,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:56:09,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:15,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:16,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:56:18,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:56:20,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:20,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:56:24,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:56:24,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:56:25,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:26,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:56:30,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:33,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:56:35,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:36,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:56:39,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:56:39,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:56:39,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:56:41,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:44,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:56:46,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:48,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:56:50,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:56:51,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:56:54,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:56:54,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:56:54,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:56:57,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:03,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:03,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:04,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:08,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:10,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:10,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:57:10,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:57:13,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:14,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:18,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:20,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:20,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:57:22,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:24,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:25,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:57:25,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:57:29,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:34,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:35,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:35,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:40,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:40,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:57:40,493 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:57:40,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:41,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 01:57:41,195 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 01:57:41,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 01:57:41,199 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 01:57:41,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 01:57:41,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:42,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:46,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:50,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:50,905 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:57:52,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:57:52,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:57:55,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:57:55,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:57:55,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:57:57,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:02,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:02,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:58:05,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:58:07,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:10,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:58:10,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:58:12,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:17,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:20,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:58:20,905 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:58:22,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:25,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:58:25,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:58:25,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:58:28,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:30,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:58:33,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:36,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:58:38,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:58:39,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:40,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:58:40,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:58:44,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:50,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:50,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:58:50,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:58:51,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:58:55,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:58:56,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:58:56,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:58:56,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:59:00,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:59:01,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:06,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:06,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:59:10,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:59:11,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:59:11,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:59:11,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:16,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:18,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:59:20,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:59:21,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:59:21,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:26,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:59:26,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:59:27,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:28,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:59:32,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:36,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:59:38,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:38,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:59:41,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:59:41,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:59:43,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:48,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 01:59:49,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:50,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 01:59:51,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 01:59:54,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 01:59:56,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 01:59:56,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 01:59:58,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:00,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:05,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:06,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:06,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:00:10,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:11,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:00:11,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:00:16,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:16,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:20,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:00:21,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:00:21,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:26,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:00:26,369 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:00:26,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:27,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:32,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:36,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:00:36,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:37,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:41,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:00:41,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:00:43,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:46,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:49,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:50,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:00:52,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:00:54,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:00:55,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:00:56,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:00:56,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:01:00,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:04,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:01:06,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:07,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:01:11,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:11,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:01:11,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:01:12,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:01:12,779 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:01:12,780 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:01:12,782 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:01:13,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:01:14,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:01:16,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:20,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:01:21,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:22,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:01:24,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:01:26,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:01:26,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:01:27,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:33,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:34,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:01:37,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:01:38,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:41,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:01:41,547 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:01:44,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:01:44,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:49,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:50,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:01:52,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:01:54,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:01:54,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:01:56,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:01:56,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:02:00,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:02,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:02:06,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:07,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:02:11,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:02:11,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:02:11,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:12,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:02:16,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:20,912 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:02:21,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:22,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:02:22,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:02:26,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:02:26,548 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:02:27,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:30,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:02:33,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:38,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:02:38,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:40,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:02:41,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:02:41,546 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:02:44,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:50,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:50,912 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:02:50,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:02:53,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:02:55,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:02:57,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:02:57,530 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:03:01,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:01,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:06,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:08,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:03:11,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:12,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:12,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:03:12,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:03:17,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:19,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:20,913 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:03:22,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:23,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:03:27,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:03:27,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:03:28,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:29,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:33,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:38,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:38,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:03:38,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:42,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:03:42,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:03:44,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:48,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:50,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:50,914 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:03:54,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:03:55,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:03:56,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:03:57,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:03:57,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:04:00,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:06,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:06,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:04:09,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:04:11,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:13,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:04:13,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:04:16,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:04:16,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:20,914 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:04:21,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:24,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:04:24,497 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:04:24,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:04:24,501 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:04:24,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:04:24,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:04:26,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:04:27,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:28,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:04:28,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:04:32,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:36,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:04:38,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:39,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:04:43,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:43,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:04:43,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:04:44,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:04:48,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:50,916 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:04:54,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:04:54,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:04:54,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:04:58,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:04:58,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:04:59,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:04,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:05:05,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:09,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:05:10,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:13,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:05:13,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:05:14,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:05:15,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:20,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:20,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:05:24,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:05:25,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:05:26,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:28,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:05:28,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:05:31,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:33,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:05:36,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:40,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:05:42,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:43,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:05:43,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:05:43,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:05:47,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:50,917 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:05:52,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:53,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:05:55,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:05:57,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:05:59,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:05:59,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:06:01,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:03,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:08,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:10,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:06:11,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:13,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:14,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:06:14,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:06:19,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:20,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:20,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:06:24,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:25,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:06:29,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:06:29,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:06:30,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:30,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:35,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:38,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:40,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:06:41,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:44,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:06:44,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:06:46,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:48,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:50,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:06:51,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:55,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:06:56,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:06:58,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:06:59,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:06:59,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:07:02,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:07,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:10,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:07:12,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:12,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:07:14,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:07:14,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:07:18,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:20,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:07:22,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:07:24,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:25,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:07:29,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:30,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:07:30,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:07:32,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:07:34,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:39,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:40,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:07:40,948 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:07:40,948 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:07:40,951 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:07:40,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:07:41,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:07:43,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:07:44,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:45,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:07:45,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:07:50,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:07:50,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:07:53,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:07:55,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:07:56,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:00,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:08:00,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:08:01,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:03,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:08:06,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:10,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:08:12,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:13,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:08:15,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:08:15,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:08:17,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:20,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:08:21,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:08:22,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:25,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:08:27,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:30,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:08:30,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:08:31,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:08:33,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:38,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:40,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:08:41,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:08:44,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:45,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:08:45,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:08:49,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:50,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:08:51,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:08:54,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:08:57,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:08:59,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:00,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:00,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:09:00,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:09:05,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:09,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:11,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:12,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:09:15,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:09:15,933 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:09:16,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:19,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:20,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:09:22,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:27,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:09:28,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:28,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:30,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:09:30,931 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:09:34,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:36,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:39,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:42,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:09:44,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:45,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:09:45,936 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:09:46,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:50,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:50,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:09:56,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:09:56,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:09:57,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:10:00,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:10:00,936 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:10:02,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:04,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:10:07,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:12,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:12,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:10:15,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:10:15,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:10:15,936 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:10:18,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:20,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:10:23,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:25,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:10:27,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:10:28,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:31,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:10:31,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:10:34,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:35,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:10:39,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:42,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:10:43,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:10:44,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:46,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:10:46,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:10:50,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:50,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:10:51,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:10:51,768 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:10:51,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:10:51,771 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:10:52,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:10:53,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:10:55,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:10:57,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:11:00,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:01,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:11:01,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:11:03,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:11:06,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:11,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:12,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:11:13,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:11:16,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:16,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:11:16,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:11:20,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:11:21,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:23,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:11:26,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:27,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:11:31,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:11:31,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:11:31,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:11:32,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:37,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:41,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:11:42,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:42,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:11:46,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:11:46,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:11:48,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:50,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:11:51,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:11:53,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:11:58,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:11:58,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:01,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:01,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:12:01,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:12:04,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:08,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:09,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:13,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:12:14,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:16,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:12:16,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:12:18,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:19,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:20,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:12:24,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:28,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:12:29,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:30,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:31,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:12:31,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:12:35,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:39,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:40,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:43,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:12:45,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:46,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:12:46,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:12:47,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:50,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:12:50,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:55,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:12:57,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:12:58,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:13:00,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:01,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:13:01,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:13:06,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:07,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:13:11,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:13,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:13:15,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:13:16,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:16,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:13:16,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:13:20,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:13:21,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:25,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:13:27,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:28,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:13:32,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:32,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:13:32,389 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:13:35,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:13:37,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:42,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:13:43,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:13:47,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:13:47,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:13:47,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:50,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:13:53,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:53,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:13:58,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:13:58,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:14:01,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:14:01,469 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:14:01,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:14:01,472 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:14:01,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:14:02,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:14:02,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:14:02,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:03,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:03,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:13,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:14:13,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:14,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:17,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:14:17,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:14:19,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:20,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:14:21,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:24,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:29,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:14:29,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:31,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:32,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:14:32,388 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:14:35,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:40,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:42,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:44,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:14:45,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:48,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:14:48,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:14:50,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:14:50,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:52,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:55,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:14:59,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:14:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:15:01,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:03,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:15:03,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:15:07,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:09,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:15:12,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:14,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:15:18,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:15:18,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:15:18,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:19,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:15:20,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:15:23,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:27,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:15:29,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:15:29,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:33,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:15:33,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:15:35,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:37,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:15:40,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:45,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:15:45,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:47,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:15:48,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:15:48,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:15:50,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:15:50,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:55,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:15:57,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:00,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:16:00,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:04,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:16:04,132 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:16:06,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:07,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:11,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:15,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:16:15,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:16,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:19,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:16:19,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:16:20,930 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:16:21,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:25,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:27,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:30,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:16:32,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:34,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:16:34,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:16:35,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:37,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:42,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:45,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:16:45,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:47,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:49,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:16:49,132 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:16:50,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:16:52,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:16:53,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:16:57,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:01,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:17:03,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:03,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:04,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:17:04,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:17:08,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:11,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:17:11,841 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:17:11,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:17:11,844 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:17:11,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:17:13,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:13,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:14,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:16,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:17:18,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:20,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:17:20,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:17:20,932 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:17:22,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:24,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:31,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:17:32,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:34,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:35,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:17:35,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:17:39,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:42,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:44,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:46,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:17:49,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:17:50,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:17:50,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:17:50,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:17:52,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:17:54,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:00,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:01,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:18:02,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:05,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:18:05,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:18:06,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:09,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:11,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:16,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:17,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:18:19,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:20,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:18:20,180 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:18:20,932 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:18:21,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:26,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:29,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:32,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:18:32,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:35,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:18:35,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:18:37,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:39,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:42,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:47,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:18:47,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:47,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:50,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:18:50,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:18:50,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:18:52,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:18:57,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:18:58,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:02,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:19:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:19:05,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:19:07,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:19:08,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:13,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:17,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:19:17,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:19:19,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:20,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:19:20,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:19:20,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:19:24,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:27,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:19:30,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:19:35,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:19:35,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:19:35,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:19:36,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:41,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:45,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:19:47,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:48,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:19:50,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:19:50,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:19:50,934 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:19:52,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:19:56,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:19:58,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:03,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:20:04,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:04,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:20:05,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:20:09,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:14,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:14,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:18,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:20:19,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:20,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:20:21,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:20:21,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:20:22,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:20:22,623 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:20:22,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:20:22,627 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:20:23,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:20:24,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:25,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:25,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:30,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:33,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:20:34,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:36,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:36,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:20:36,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:20:41,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:42,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:46,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:49,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:20:50,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:20:51,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:20:51,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:20:51,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:20:52,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:20:56,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:02,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:21:02,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:04,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:21:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:21:06,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:21:07,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:12,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:17,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:18,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:21:19,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:21:20,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:21:21,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:21:21,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:21:23,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:25,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:21:29,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:34,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:34,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:21:35,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:21:36,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:21:36,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:21:39,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:44,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:45,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:21:49,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:21:50,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:50,937 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:21:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:21:51,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:21:55,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:21:55,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:00,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:04,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:22:05,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:05,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:06,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:22:06,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:22:10,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:15,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:16,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:19,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:22:20,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:20,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:22:21,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:22:21,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:22:24,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:26,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:31,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:34,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:34,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:22:36,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:36,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:22:36,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:22:41,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:44,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:46,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:50,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:22:50,940 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:22:51,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:22:51,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:22:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:22:52,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:22:57,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:02,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:02,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:05,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:23:06,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:23:06,981 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:23:08,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:12,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:14,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:19,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:20,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:23:20,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:20,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:23:21,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:23:21,978 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:23:24,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:29,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:30,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:34,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:35,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:23:37,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:23:37,583 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:23:38,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:23:38,767 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:23:38,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:23:38,770 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:23:39,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:23:39,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:40,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:40,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:45,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:50,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:23:50,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:23:50,940 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:23:50,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:52,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:23:52,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:23:56,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:23:58,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:24:02,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:06,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:24:07,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:24:07,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:24:07,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:08,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:24:12,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:17,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:24:17,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:20,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:24:21,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:24:22,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:24:22,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:24:23,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:27,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:24:29,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:34,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:36,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:24:37,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:24:37,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:24:38,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:24:39,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:44,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:46,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:24:50,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:50,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:24:51,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:24:53,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:24:53,118 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:24:55,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:24:56,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:00,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:04,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:05,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:06,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:25:08,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:25:08,117 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:25:11,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:14,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:17,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:20,943 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:25:21,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:25:22,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:23,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:25:23,117 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:25:24,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:28,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:33,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:34,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:36,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:25:38,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:25:38,119 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:25:39,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:44,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:45,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:50,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:25:50,943 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:25:51,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:25:52,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:25:53,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:25:53,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:25:55,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:00,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:02,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:05,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:06,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:26:08,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:26:08,852 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:26:10,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:12,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:15,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:20,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:20,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:26:21,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:26:21,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:23,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:26:23,850 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:26:27,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:30,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:32,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:37,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:37,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:26:38,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:26:38,850 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:26:40,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:43,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:47,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:26:47,553 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:26:47,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:26:47,556 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:26:47,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:26:48,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:48,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:49,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:50,944 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:26:52,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:26:53,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:26:53,851 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:26:54,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:26:58,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:26:59,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:04,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:07,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:27:08,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:27:08,851 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:27:09,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:27:10,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:15,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:16,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:27:20,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:20,944 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:27:22,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:27:24,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:27:24,229 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:27:26,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:27:26,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:31,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:36,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:27:36,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:37,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:27:39,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:27:39,227 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:27:42,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:44,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:27:47,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:50,946 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:27:52,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:27:52,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:27:54,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:27:54,227 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:27:54,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:27:58,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:03,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:04,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:28:07,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:28:08,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:09,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:28:09,227 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:28:14,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:28:14,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:19,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:20,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:28:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:28:24,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:28:24,228 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:28:24,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:28:25,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:30,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:32,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:28:36,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:37,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:28:40,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:28:40,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:28:41,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:42,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:28:46,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:50,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:28:51,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:28:52,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:28:52,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:28:55,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:28:55,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:28:57,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:02,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:29:02,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:07,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:29:07,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:10,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:29:10,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:29:12,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:29:13,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:18,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:20,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:29:20,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:29:22,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:29:23,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:25,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:29:25,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:29:29,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:31,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:29:34,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:37,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:29:39,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:40,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:29:40,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:29:41,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:29:44,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:50,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:50,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:29:51,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:29:52,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:29:55,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:29:56,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:29:56,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:29:57,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:29:57,738 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:29:57,738 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:29:57,741 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:29:58,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:29:59,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:00,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:01,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:06,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:07,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:30:09,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:11,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:30:11,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:30:11,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:17,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:18,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:20,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:30:22,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:30:22,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:26,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:26,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:30:26,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:30:28,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:33,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:36,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:37,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:30:39,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:41,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:30:41,676 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:30:44,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:46,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:50,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:30:53,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:30:55,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:30:56,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:30:56,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:30:56,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:31:01,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:06,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:31:07,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:08,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:31:12,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:12,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:31:12,921 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:31:14,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:31:17,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:20,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:31:22,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:23,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:31:24,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:31:27,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:31:27,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:31:28,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:33,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:34,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:31:38,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:38,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:31:42,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:31:42,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:31:43,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:31:44,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:49,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:50,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:31:53,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:31:53,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:31:54,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:31:57,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:31:57,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:32:00,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:03,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:32:05,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:09,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:32:10,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:12,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:32:12,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:32:13,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:32:16,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:20,954 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:32:21,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:32:21,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:24,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:32:26,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:28,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:32:28,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:32:31,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:32:32,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:37,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:39,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:32:41,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:32:42,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:43,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:32:43,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:32:48,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:50,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:32:51,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:32:53,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:54,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:32:58,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:32:58,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:32:59,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:32:59,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:04,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:07,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:33:07,707 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:33:07,707 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:33:07,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:33:08,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:33:08,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:09,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:33:09,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:10,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:13,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:33:13,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:33:15,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:18,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:20,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:20,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:33:25,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:33:25,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:26,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:28,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:33:28,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:33:31,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:36,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:38,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:40,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:33:41,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:43,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:33:43,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:33:46,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:46,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:50,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:33:52,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:33:55,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:33:56,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:33:58,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:33:58,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:33:58,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:03,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:06,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:34:09,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:10,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:34:13,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:34:13,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:34:14,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:17,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:34:19,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:20,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:34:24,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:25,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:34:27,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:34:28,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:34:28,522 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:34:30,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:36,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:37,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:34:40,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:34:41,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:43,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:34:43,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:34:45,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:34:47,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:50,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:34:52,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:34:55,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:34:55,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:34:58,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:34:58,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:34:58,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:04,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:05,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:35:09,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:10,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:35:13,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:35:13,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:35:14,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:15,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:35:19,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:20,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:35:24,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:25,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:35:28,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:35:28,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:35:29,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:35:30,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:35,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:39,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:35:40,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:35:41,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:43,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:35:43,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:35:46,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:47,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:35:50,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:35:51,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:55,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:35:57,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:35:57,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:35:58,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:35:58,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:36:03,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:07,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:09,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:10,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:36:13,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:36:13,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:36:15,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:16,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:20,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:20,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:36:24,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:36:24,442 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:36:24,442 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:36:24,445 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:36:24,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:36:24,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:25,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:25,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:36:26,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:28,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:36:28,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:36:31,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:34,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:36,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:40,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:36:41,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:43,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:36:43,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:36:45,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:47,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:50,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:36:52,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:53,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:36:55,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:36:57,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:36:58,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:36:58,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:37:03,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:37:03,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:08,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:10,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:37:13,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:37:13,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:13,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:37:13,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:37:19,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:20,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:37:23,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:37:24,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:25,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:37:28,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:37:28,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:37:30,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:35,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:40,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:40,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:37:41,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:37:43,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:37:43,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:37:46,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:50,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:37:51,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:37:51,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:55,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:37:56,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:37:58,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:37:58,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:37:59,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:02,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:07,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:09,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:10,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:38:12,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:13,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:38:13,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:38:18,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:19,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:20,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:38:23,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:26,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:38:28,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:38:28,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:38:29,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:29,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:34,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:39,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:39,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:41,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:38:44,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:45,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:38:45,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:38:49,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:49,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:50,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:38:54,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:38:56,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:38:56,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:38:59,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:39:00,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:39:05,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:07,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:10,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:11,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:39:15,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:39:15,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:39:15,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:17,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:20,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:20,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:39:25,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:26,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:39:27,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:30,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:39:30,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:39:31,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:35,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:37,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:41,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:39:42,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:44,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:39:44,153 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:39:44,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:39:44,156 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:39:44,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:39:45,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:45,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:39:45,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:39:46,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:47,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:50,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:39:52,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:39:55,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:39:56,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:39:58,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:00,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:40:00,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:40:03,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:05,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:08,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:11,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:40:14,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:15,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:40:15,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:40:15,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:19,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:20,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:40:24,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:25,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:26,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:40:29,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:31,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:40:31,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:40:33,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:35,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:40,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:40,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:41,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:41,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:40:45,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:47,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:40:47,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:40:47,235 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:40:49,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:40:50,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:40:51,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:56,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:40:57,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:40:59,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:41:02,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:41:02,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:41:02,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:07,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:09,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:41:12,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:41:12,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:17,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:41:17,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:41:17,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:41:18,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:19,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:41:20,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:41:23,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:27,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:41:28,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:41:29,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:32,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:41:32,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:41:32,234 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:41:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:40,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:41:40,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:42,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:41:45,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:47,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:41:47,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:41:47,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:41:50,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:41:50,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:56,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:41:57,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:41:57,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:01,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:02,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:42:02,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:42:02,236 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:42:07,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:07,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:12,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:13,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:42:15,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:17,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:42:17,234 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:42:17,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:20,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:42:22,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:25,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:28,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:28,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:42:33,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:33,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:42:33,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:42:33,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:42:35,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:38,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:43,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:42:43,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:45,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:48,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:42:48,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:42:48,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:50,969 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:42:53,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:42:54,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:42:54,015 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:42:54,016 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:42:54,018 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:42:54,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:42:54,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:55,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:42:58,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:42:59,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:03,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:43:03,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:43:04,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:05,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:43:09,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:13,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:43:15,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:15,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:43:18,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:43:18,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:43:20,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:20,969 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:43:25,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:43:26,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:28,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:43:31,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:33,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:43:33,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:43:35,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:43:36,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:41,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:43,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:43:46,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:43:47,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:48,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:43:48,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:43:50,971 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:43:52,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:54,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:43:57,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:43:58,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:02,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:03,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:44:03,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:44:04,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:44:07,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:13,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:14,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:44:18,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:19,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:19,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:44:19,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:44:20,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:44:22,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:44:23,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:28,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:29,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:32,325 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:44:34,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:34,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:44:34,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:44:34,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:39,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:41,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:44:44,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:45,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:49,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:44:49,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:44:49,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:44:50,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:50,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:44:51,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:44:55,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:44:59,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:01,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:01,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:04,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:04,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:45:04,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:45:06,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:11,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:12,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:14,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:17,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:19,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:19,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:19,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:45:19,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:45:20,977 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:45:22,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:27,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:29,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:29,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:32,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:34,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:34,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:45:34,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:45:38,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:39,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:43,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:45,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:45:47,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:48,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:49,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:45:49,671 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:45:50,977 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:45:53,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:45:57,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:45:59,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:00,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:46:04,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:05,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:46:05,069 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:46:06,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:46:06,275 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:46:06,276 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:46:06,278 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:46:07,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:46:07,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:08,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:10,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:15,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:15,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:46:18,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:20,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:46:20,068 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:46:20,978 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:46:20,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:26,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:28,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:30,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:46:31,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:35,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:46:35,069 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:46:36,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:38,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:41,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:45,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:46:46,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:46,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:50,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:46:50,069 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:46:50,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:46:51,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:46:56,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:46:57,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:02,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:02,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:47:05,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:47:05,069 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:47:06,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:47:07,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:12,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:14,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:47:17,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:47:17,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:20,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:47:21,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:47:21,488 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:47:23,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:47:23,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:28,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:32,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:47:33,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:47:34,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:36,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:47:36,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:47:39,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:43,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:47:44,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:47,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:47:49,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:47:50,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:47:51,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:47:51,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:47:53,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:47:55,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:00,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:01,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:48:02,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:48:05,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:06,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:48:06,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:48:11,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:11,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:48:16,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:18,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:48:20,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:48:21,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:48:21,488 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:48:21,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:21,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:48:26,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:32,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:48:32,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:33,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:48:36,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:48:36,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:48:37,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:42,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:48:43,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:48,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:48,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:48:50,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:48:50,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:48:51,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:48:51,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:48:53,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:48:59,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:00,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:03,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:49:04,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:06,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:49:06,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:49:09,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:10,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:14,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:17,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:49:17,064 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:49:17,065 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:49:17,067 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:49:17,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:49:18,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:18,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:49:19,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:20,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:20,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:49:21,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:49:21,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:49:25,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:28,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:30,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:35,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:49:35,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:36,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:49:36,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:49:38,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:41,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:46,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:50,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:49:50,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:49:51,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:49:51,488 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:49:51,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:49:52,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:49:56,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:01,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:02,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:50:05,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:50:06,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:07,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:50:07,887 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:50:10,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:50:12,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:17,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:19,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:50:20,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:50:20,984 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:50:22,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:50:22,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:50:23,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:28,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:29,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:50:33,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:36,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:50:37,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:50:37,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:50:39,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:39,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:50:44,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:49,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:50,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:50:50,984 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:50:51,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:50:52,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:50:52,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:50:55,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:50:58,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:51:01,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:06,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:06,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:51:07,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:51:07,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:51:08,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:51:12,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:17,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:18,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:51:20,984 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:51:21,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:51:22,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:51:22,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:51:23,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:28,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:51:29,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:34,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:36,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:51:37,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:51:37,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:51:38,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:51:39,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:44,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:46,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:51:49,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:50,985 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:51:53,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:51:53,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:51:53,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:51:53,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:51:55,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:51:56,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:00,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:05,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:06,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:08,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:52:08,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:52:08,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:52:11,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:16,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:16,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:20,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:52:21,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:23,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:52:23,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:52:23,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:52:24,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:27,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:32,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:32,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:52:32,884 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:52:32,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:52:32,887 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:52:33,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:52:34,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:35,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:37,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:38,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:52:38,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:52:38,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:52:43,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:44,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:48,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:50,986 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:52:52,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:52:53,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:52:53,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:52:53,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:52:53,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:52:58,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:01,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:53:03,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:08,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:09,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:53:09,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:53:09,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:53:09,584 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:53:12,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:53:13,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:18,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:20,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:53:22,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:53:23,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:24,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:53:24,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:53:24,583 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:53:29,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:32,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:53:34,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:39,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:53:39,582 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:53:40,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:53:40,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:45,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:50,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:53:50,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:53:50,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:53:54,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:53:54,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:53:54,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:53:54,583 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:53:56,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:00,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:54:02,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:07,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:09,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:54:09,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:54:09,584 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:54:09,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:54:10,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:54:12,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:18,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:20,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:54:20,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:54:23,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:24,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:54:24,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:54:24,583 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:54:29,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:34,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:39,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:54:39,583 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:54:40,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:45,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:50,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:54:50,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:54:54,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:54:54,584 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:54:54,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:54:55,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:00,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:04,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:55:06,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:10,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:55:10,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:55:10,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:55:10,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:55:11,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:14,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:55:16,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:20,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:55:22,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:22,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:55:25,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:55:25,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:55:25,739 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:55:27,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:33,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:55:33,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:38,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:40,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:55:40,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:55:40,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:55:43,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:55:43,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:48,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:50,991 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:55:53,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:55:53,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:55:55,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:55:55,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:55:55,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:55:55,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:55:59,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:00,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:04,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:09,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:56:09,252 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:56:09,253 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:56:09,256 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:56:09,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:56:10,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:10,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:10,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:56:10,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:56:10,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:56:10,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:56:12,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:15,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:20,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:20,991 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:56:20,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:25,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:26,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:56:26,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:56:26,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:56:26,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:56:30,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:31,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:36,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:38,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:41,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:56:41,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:56:41,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:56:41,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:46,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:48,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:56:50,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:56:51,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:56,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:56:56,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:56:56,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:56:57,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:56:58,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:02,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:08,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:08,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:11,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:57:11,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:57:13,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:18,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:19,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:20,993 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:57:24,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:26,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:57:26,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:57:28,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:30,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:35,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:38,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:41,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:41,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:57:41,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:57:46,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:46,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:50,993 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:57:51,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:57:56,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:57:56,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:57:57,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:57:57,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:02,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:07,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:58:08,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:12,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:58:12,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:58:12,747 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:58:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:15,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:58:18,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:20,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:58:24,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:25,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:58:27,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:58:27,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:58:27,746 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:58:29,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:35,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:35,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:58:40,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:42,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:58:42,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:58:42,747 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:58:45,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:58:45,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:50,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:50,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:58:55,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:58:56,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:58:57,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:58:57,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:58:57,747 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:59:01,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:02,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:06,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:12,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:12,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:12,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:59:12,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:59:12,747 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:59:17,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:20,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:59:21,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 02:59:21,359 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 02:59:21,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 02:59:21,362 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 02:59:21,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 02:59:22,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:23,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:23,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:27,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:59:27,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:59:27,751 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:59:28,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:32,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:34,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:39,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:42,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:42,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:59:42,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:59:42,759 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 02:59:44,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:50,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:50,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 02:59:50,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 02:59:55,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 02:59:57,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:59:57,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 02:59:57,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 02:59:57,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:00:00,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:00:01,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:06,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:10,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:00:11,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:12,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:00:12,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:00:12,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:00:17,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:20,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:00:20,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:00:22,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:27,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:00:27,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:00:27,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:00:28,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:33,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:38,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:42,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:00:42,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:00:42,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:00:42,924 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:00:43,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:48,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:50,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:00:53,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:00:54,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:59,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:00:59,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:00:59,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:00:59,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:01:03,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:01:04,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:09,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:13,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:01:14,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:01:14,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:01:15,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:20,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:21,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:01:23,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:01:26,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:29,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:01:29,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:01:29,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:01:29,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:01:31,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:33,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:01:36,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:42,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:43,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:01:44,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:01:44,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:01:44,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:01:44,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:01:47,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:50,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:01:51,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:01:53,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:58,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:01:59,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:01:59,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:01:59,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:02:01,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:03,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:09,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:10,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:14,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:02:14,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:02:14,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:02:14,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:20,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:20,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:21,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:02:26,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:28,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:29,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:02:29,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:02:29,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:02:29,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:02:31,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:36,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:38,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:42,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:46,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:02:46,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:02:46,142 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:02:48,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:48,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:51,001 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:02:54,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:02:56,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:02:56,557 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:02:56,558 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:02:56,560 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:02:57,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:02:58,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:59,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:02:59,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:01,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:03:01,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:03:01,139 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:03:05,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:09,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:03:10,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:15,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:16,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:03:16,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:03:16,139 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:03:17,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:03:21,001 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:03:21,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:26,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:27,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:03:31,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:31,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:03:31,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:03:31,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:03:36,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:37,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:03:41,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:45,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:03:46,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:03:46,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:03:46,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:03:47,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:51,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:03:53,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:03:55,371 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:03:58,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:01,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:01,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:01,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:04:01,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:04:04,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:05,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:04:09,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:14,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:15,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:04:16,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:16,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:16,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:04:16,938 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:04:19,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:21,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:04:25,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:29,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:04:30,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:31,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:31,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:31,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:04:31,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:04:36,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:39,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:04:42,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:46,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:46,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:04:46,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:04:46,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:04:47,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:49,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:04:51,004 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:04:53,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:04:57,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:04:58,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:01,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:05:01,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:05:01,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:05:04,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:06,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:05:09,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:14,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:16,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:05:16,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:05:16,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:05:16,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:05:20,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:21,004 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:05:24,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:05:25,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:30,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:33,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:05:33,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:05:33,054 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:05:35,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:05:36,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:41,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:45,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:05:46,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:48,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:05:48,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:05:48,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:05:51,006 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:05:52,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:05:55,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:05:57,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:02,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:03,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:06:03,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:06:03,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:07,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:11,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:06:11,841 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:06:11,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:06:11,843 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:06:12,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:06:12,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:13,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:14,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:17,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:18,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:06:18,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:06:21,006 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:06:23,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:23,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:28,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:33,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:06:33,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:06:33,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:34,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:39,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:43,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:44,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:48,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:06:48,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:06:48,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:06:48,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:06:50,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:06:51,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:06:51,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:06:56,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:01,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:01,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:07:03,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:07:03,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:07:03,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:07:03,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:07:07,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:11,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:07:13,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:18,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:18,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:07:18,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:07:18,953 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:07:19,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:07:21,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:07:24,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:29,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:29,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:07:33,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:07:33,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:07:33,952 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:07:35,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:39,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:07:40,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:45,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:48,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:07:48,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:07:48,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:07:48,952 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:07:51,008 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:07:51,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:56,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:07:57,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:01,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:05,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:08:05,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:08:05,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:08:05,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:08:07,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:07,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:12,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:17,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:20,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:08:20,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:08:20,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:08:21,008 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:08:23,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:27,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:28,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:33,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:35,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:35,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:08:35,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:08:35,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:08:38,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:43,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:45,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:48,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:50,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:08:50,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:08:50,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:08:50,320 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:08:51,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:08:54,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:08:55,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:08:59,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:04,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:05,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:09:05,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:09:05,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:09:05,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:09,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:13,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:14,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:19,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:20,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:09:20,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:09:20,980 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:09:21,183 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:09:22,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:09:22,133 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:09:22,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:09:22,136 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:09:22,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:09:23,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:24,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:25,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:30,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:33,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:35,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:35,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:09:35,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:09:35,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:09:41,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:43,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:46,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:50,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:09:50,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:09:50,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:09:51,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:09:51,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:09:51,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:09:56,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:01,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:01,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:10:05,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:10:05,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:10:05,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:10:07,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:12,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:10:12,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:17,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:20,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:10:20,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:10:20,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:10:20,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:10:21,163 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:10:23,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:28,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:30,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:10:33,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:36,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:10:36,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:10:36,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:10:36,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:10:38,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:40,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:10:43,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:49,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:10:49,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:51,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:10:51,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:10:51,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:10:51,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:10:51,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:10:54,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:10:59,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:00,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:05,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:06,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:11:06,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:11:06,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:11:06,774 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:11:09,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:10,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:15,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:17,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:20,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:21,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:11:21,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:11:21,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:11:21,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:11:25,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:27,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:30,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:36,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:36,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:11:36,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:11:36,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:11:37,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:42,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:47,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:47,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:51,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:11:51,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:11:51,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:11:51,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:11:51,776 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:11:52,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:11:55,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:11:58,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:03,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:05,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:06,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:12:06,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:12:06,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:12:09,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:14,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:15,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:19,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:21,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:12:21,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:12:21,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:12:21,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:12:21,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:12:23,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:24,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:29,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:32,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:12:32,348 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:12:32,348 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:12:32,350 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:12:33,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:12:34,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:35,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:35,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:36,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:12:36,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:12:36,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:12:40,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:44,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:46,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:51,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:12:51,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:12:51,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:12:51,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:12:51,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:12:54,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:12:56,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:02,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:13:02,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:06,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:06,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:06,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:13:06,780 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:13:07,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:12,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:13:13,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:18,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:21,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:13:21,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:21,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:13:21,780 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:13:22,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:13:23,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:28,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:32,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:13:34,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:38,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:38,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:38,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:13:38,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:13:40,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:40,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:13:45,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:49,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:13:50,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:51,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:13:53,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:53,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:13:53,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:13:53,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:13:56,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:13:59,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:02,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:07,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:08,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:08,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:08,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:14:08,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:14:09,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:12,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:17,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:19,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:21,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:14:23,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:23,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:23,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:23,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:14:23,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:14:27,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:28,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:33,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:37,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:38,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:38,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:38,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:14:38,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:14:38,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:43,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:47,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:48,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:51,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:14:54,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:14:54,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:54,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:14:54,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:14:54,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:14:57,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:14:59,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:04,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:06,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:09,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:15:09,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:15:09,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:15:09,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:15,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:16,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:20,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:21,020 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:15:24,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:15:24,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:15:24,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:15:25,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:26,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:30,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:35,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:36,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:39,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:15:39,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:15:39,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:15:41,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:43,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:15:43,427 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:15:43,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:15:43,431 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:15:44,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:15:44,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:45,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:47,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:51,021 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:15:52,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:15:54,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:15:54,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:15:54,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:15:58,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:04,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:04,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:16:09,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:10,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:10,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:10,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:16:10,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:16:14,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:16:14,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:19,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:21,021 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:16:22,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:16:25,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:25,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:25,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:25,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:16:25,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:16:30,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:32,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:16:35,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:40,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:40,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:40,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:16:40,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:16:40,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:41,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:16:45,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:51,022 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:16:51,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:51,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:16:55,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:16:55,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:16:55,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:16:56,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:16:59,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:17:02,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:07,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:09,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:17:10,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:17:10,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:17:10,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:17:12,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:17,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:20,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:17:21,022 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:17:23,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:27,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:17:27,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:17:27,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:17:27,099 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:17:28,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:30,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:17:33,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:38,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:40,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:17:42,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:17:42,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:17:42,098 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:17:44,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:48,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:17:49,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:51,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:17:55,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:17:57,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:17:57,098 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:17:58,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:18:00,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:05,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:08,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:18:10,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:12,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:18:12,098 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:18:16,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:18,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:18:21,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:18:22,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:27,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:18:27,099 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:18:27,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:28,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:18:32,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:37,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:42,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:18:42,099 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:18:42,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:18:43,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:48,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:51,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:18:52,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:18:54,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:18:58,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:18:58,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:18:58,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:18:58,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:18:59,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:00,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:19:00,504 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:19:00,505 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:19:00,508 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:19:00,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:19:01,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:02,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:05,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:10,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:12,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:13,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:19:13,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:19:13,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:19:13,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:19:16,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:20,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:21,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:19:22,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:27,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:28,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:19:28,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:19:28,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:19:28,601 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:19:30,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:32,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:37,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:40,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:42,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:43,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:19:43,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:19:43,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:19:47,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:47,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:51,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:19:53,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:19:58,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:19:58,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:19:58,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:19:58,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:19:58,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:03,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:08,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:20:09,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:14,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:20:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:20:14,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:20:18,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:20:19,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:21,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:20:25,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:26,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:20:29,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:20:29,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:20:29,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:20:30,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:35,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:36,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:20:40,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:44,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:20:44,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:20:44,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:20:46,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:20:46,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:51,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:20:52,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:56,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:20:57,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:20:59,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:20:59,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:21:02,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:04,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:21:08,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:13,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:21:14,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:21:14,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:21:19,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:21,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:21:24,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:21:25,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:30,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:30,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:21:30,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:21:30,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:21:30,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:21:34,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:21:35,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:44,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:21:45,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:45,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:21:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:21:45,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:21:45,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:21:50,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:21:51,030 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:21:54,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:21:56,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:01,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:01,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:22:01,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:22:01,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:22:04,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:07,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:12,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:12,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:22:12,661 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:22:12,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:22:12,665 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:22:13,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:22:14,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:15,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:16,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:22:16,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:22:16,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:22:18,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:21,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:22:23,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:23,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:28,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:31,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:22:31,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:22:31,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:22:33,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:34,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:39,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:42,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:44,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:46,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:22:46,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:22:46,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:22:49,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:22:50,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:22:51,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:22:55,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:00,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:00,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:23:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:23:01,831 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:23:06,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:10,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:11,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:16,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:17,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:23:17,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:23:17,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:23:17,132 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:23:20,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:21,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:23:22,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:27,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:28,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:32,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:32,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:23:32,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:23:32,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:23:32,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:23:37,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:38,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:42,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:47,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:23:47,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:23:47,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:23:47,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:48,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:51,032 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:23:53,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:23:56,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:23:58,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:02,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:24:02,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:24:02,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:24:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:06,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:24:10,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:15,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:16,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:24:17,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:24:17,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:24:17,132 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:24:21,034 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:24:21,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:26,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:26,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:24:31,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:32,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:24:32,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:24:32,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:24:32,481 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:24:34,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:24:36,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:41,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:45,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:24:47,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:47,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:24:47,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:24:51,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:24:52,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:24:55,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:24:57,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:02,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:25:02,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:25:02,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:05,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:07,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:13,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:13,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:17,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:25:17,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:25:18,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:21,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:25:21,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:25:21,152 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:25:21,153 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:25:21,156 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:25:21,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:25:21,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:23,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:24,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:29,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:32,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:25:32,480 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:25:33,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:34,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:39,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:42,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:44,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:48,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:25:48,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:25:48,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:25:48,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:25:50,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:25:50,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:25:51,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:25:55,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:00,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:00,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:03,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:26:03,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:26:03,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:26:06,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:10,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:11,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:16,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:26:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:26:18,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:26:20,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:21,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:26:22,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:27,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:28,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:32,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:33,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:26:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:26:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:26:33,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:26:38,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:38,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:43,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:48,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:26:48,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:26:48,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:49,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:51,036 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:26:54,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:26:56,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:26:59,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:04,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:04,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:27:04,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:27:04,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:27:06,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:27:09,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:14,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:17,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:27:19,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:27:19,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:27:19,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:27:19,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:21,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:27:25,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:27,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:27:30,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:34,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:27:34,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:27:34,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:27:34,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:27:35,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:37,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:27:40,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:45,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:27:46,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:49,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:27:49,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:27:49,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:27:51,039 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:27:52,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:27:55,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:27:57,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:02,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:04,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:28:04,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:28:04,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:28:05,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:28:07,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:12,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:15,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:28:17,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:21,040 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:28:21,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:28:21,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:28:21,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:28:21,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:28:23,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:23,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:28:28,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:31,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:28:31,581 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:28:31,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:28:31,583 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:28:32,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:28:33,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:28:33,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:36,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:28:36,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:28:36,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:28:36,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:28:39,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:42,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:28:44,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:49,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:28:50,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:28:51,040 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:28:51,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:28:51,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:28:51,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:28:55,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:00,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:00,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:05,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:06,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:29:06,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:29:06,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:29:10,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:11,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:16,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:18,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:21,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:29:21,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:29:21,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:29:21,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:29:21,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:26,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:28,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:31,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:36,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:29:36,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:29:36,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:29:36,147 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:29:37,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:38,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:42,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:47,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:47,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:51,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:29:51,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:29:51,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:29:51,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:29:52,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:29:57,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:29:58,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:03,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:06,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:30:06,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:30:06,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:30:07,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:30:08,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:13,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:17,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:30:19,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:21,043 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:30:21,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:30:21,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:30:21,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:30:24,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:25,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:30:29,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:35,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:35,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:30:36,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:30:36,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:30:36,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:30:40,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:45,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:45,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:30:50,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:30:51,056 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:30:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:30:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:30:52,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:30:52,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:30:55,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:30:56,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:01,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:05,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:31:07,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:07,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:31:07,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:31:07,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:31:12,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:13,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:31:17,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:21,045 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:31:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:31:22,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:31:22,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:31:22,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:31:22,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:23,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:31:27,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:32,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:33,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:31:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:31:37,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:31:37,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:31:38,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:43,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:48,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:50,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:31:50,464 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:31:50,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:31:50,467 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:31:50,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:31:50,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:31:51,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:31:51,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:31:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:31:52,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:31:52,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:31:54,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:31:59,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:00,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:32:04,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:08,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:32:08,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:32:08,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:32:10,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:11,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:32:15,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:19,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:32:20,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:21,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:32:23,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:32:23,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:32:23,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:32:25,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:29,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:32:31,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:36,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:38,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:32:38,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:32:38,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:32:39,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:32:41,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:46,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:49,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:32:51,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:32:52,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:32:53,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:32:53,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:32:53,108 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:32:57,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:01,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:33:02,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:07,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:08,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:33:08,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:33:08,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:33:11,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:33:12,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:17,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:21,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:33:21,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:33:23,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:23,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:33:23,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:33:23,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:33:28,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:31,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:33:33,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:38,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:38,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:33:38,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:33:38,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:33:38,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:33:41,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:33:43,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:48,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:51,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:33:51,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:33:53,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:33:53,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:33:54,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:59,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:33:59,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:04,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:08,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:34:08,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:34:09,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:09,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:14,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:19,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:20,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:21,048 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:34:23,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:34:23,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:34:23,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:34:23,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:34:26,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:27,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:31,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:37,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:38,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:38,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:34:38,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:34:38,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:34:43,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:47,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:48,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:51,049 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:34:53,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:34:53,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:34:53,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:34:53,842 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:34:54,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:34:55,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:34:59,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:04,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:35:04,299 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:35:04,300 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:35:04,302 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:35:05,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:35:05,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:05,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:06,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:08,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:35:08,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:35:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:15,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:16,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:21,049 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:35:22,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:23,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:35:23,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:35:25,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:27,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:32,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:35,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:37,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:38,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:35:38,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:35:42,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:43,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:47,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:51,051 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:35:52,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:35:53,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:35:53,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:35:53,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:35:58,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:03,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:03,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:36:08,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:10,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:36:10,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:36:10,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:36:10,268 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:36:13,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:13,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:36:18,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:21,051 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:36:21,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:36:24,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:25,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:36:25,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:36:25,250 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:36:29,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:31,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:36:35,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:40,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:36:40,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:36:40,250 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:36:40,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:41,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:36:45,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:50,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:36:51,052 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:36:52,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:36:55,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:36:55,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:36:55,250 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:36:56,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:00,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:37:02,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:07,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:10,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:37:10,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:37:10,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:37:10,250 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:37:12,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:20,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:37:21,053 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:37:23,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:26,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:37:26,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:37:26,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:37:28,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:29,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:37:33,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:38,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:41,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:37:41,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:37:41,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:37:43,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:47,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:37:49,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:51,053 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:37:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:37:56,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:37:56,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:37:56,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:37:57,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:38:00,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:06,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:07,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:38:11,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:38:11,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:38:11,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:38:11,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:17,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:17,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:38:21,054 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:38:23,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:24,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:38:24,750 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:38:24,751 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:38:24,753 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:38:24,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:38:25,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:38:26,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:38:26,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:38:26,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:38:26,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:38:27,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:38:28,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:33,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:38,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:41,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:38:41,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:38:41,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:38:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:49,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:51,054 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:38:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:38:56,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:38:56,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:38:56,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:39:00,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:05,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:10,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:11,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:39:11,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:39:11,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:39:16,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:21,055 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:39:22,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:26,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:39:26,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:39:26,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:39:27,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:32,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:37,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:41,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:39:41,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:39:41,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:39:42,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:47,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:51,055 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:39:53,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:39:56,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:39:56,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:39:56,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:39:58,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:03,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:08,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:11,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:40:11,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:40:11,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:40:14,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:19,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:21,056 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:40:25,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:26,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:40:26,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:40:26,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:40:30,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:35,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:40,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:41,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:40:41,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:40:41,550 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:40:46,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:51,056 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:40:52,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:40:56,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:40:56,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:40:56,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:40:57,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:02,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:07,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:11,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:41:11,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:41:11,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:41:13,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:18,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:21,057 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:41:24,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:26,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:41:26,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:41:26,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:41:29,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:34,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:39,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:41,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:41:41,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:41:41,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:41:45,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:50,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:51,058 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:41:56,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:41:56,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:41:56,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:41:56,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:42:01,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:06,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:11,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:42:11,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:42:11,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:42:12,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:17,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:21,058 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:42:23,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:26,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:42:26,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:42:26,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:42:28,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:33,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:38,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:41,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:42:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:42:41,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:42:43,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:48,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:51,058 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:42:54,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:42:56,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:42:56,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:42:56,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:42:59,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:04,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:09,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:11,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:43:11,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:43:11,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:43:14,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:19,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:21,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:43:25,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:26,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:43:26,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:43:26,541 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:43:30,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:35,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:40,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:43:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:43:41,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:43:45,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:50,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:51,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:43:56,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:43:56,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:43:56,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:43:56,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:44:01,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:06,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:11,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:44:11,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:44:11,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:44:12,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:17,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:21,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:44:23,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:26,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:44:26,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:44:26,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:44:28,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:33,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:38,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:41,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:44:41,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:44:41,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:44:44,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:49,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:51,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:44:55,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:44:56,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:44:56,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:44:56,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:45:00,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:05,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:10,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:11,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:45:11,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:45:11,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:45:16,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:21,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:45:22,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:26,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:45:26,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:45:26,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:45:27,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:32,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:37,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:41,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:45:41,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:45:41,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:45:42,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:47,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:51,061 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:45:53,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:45:56,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:45:56,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:45:56,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:45:58,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:03,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:08,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:11,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:46:11,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:46:11,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:46:13,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:18,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:21,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:46:24,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:26,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:46:26,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:46:26,543 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:46:29,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:34,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:39,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:41,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:46:41,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:46:41,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:46:44,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:49,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:51,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:46:55,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:46:56,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:46:56,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:46:56,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:47:00,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:05,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:10,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:11,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:47:11,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:47:11,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:47:16,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:21,063 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:47:22,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:26,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:47:26,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:47:26,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:47:27,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:32,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:37,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:41,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:47:41,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:47:41,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:47:42,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:47,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:51,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:47:53,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:47:56,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:47:56,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:47:56,544 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:47:58,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:03,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:08,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:11,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:48:11,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:48:11,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:48:14,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:19,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:21,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:48:25,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:26,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:48:26,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:48:26,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:48:30,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:35,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:40,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:41,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:48:41,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:48:41,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:48:46,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:49,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:48:49,369 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:48:49,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:48:49,372 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:48:49,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:48:50,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:48:51,066 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:48:52,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:48:56,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:48:56,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:48:56,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:48:57,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:02,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:07,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:11,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:49:11,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:49:11,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:49:13,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:19,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:20,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:49:21,067 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:49:25,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:27,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:49:27,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:49:27,089 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:49:30,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:35,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:40,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:41,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:49:41,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:49:45,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:50,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:51,067 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:49:56,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:49:57,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:49:57,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:49:57,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:49:59,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:50:01,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:04,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:50:06,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:07,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:50:11,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:13,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:50:13,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:50:16,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:21,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:50:22,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:27,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:31,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:50:31,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:50:31,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:50:32,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:50:32,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:50:32,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:33,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:50:37,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:42,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:46,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:50:47,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:50:47,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:50:47,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:51,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:50:53,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:50:58,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:01,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:51:02,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:51:02,059 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:51:03,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:08,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:13,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:16,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:51:17,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:51:17,059 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:51:18,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:21,069 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:51:24,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:29,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:31,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:51:32,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:51:32,059 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:51:34,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:51:35,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:40,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:44,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:51:45,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:46,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:51:47,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:51:47,059 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:51:51,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:51:51,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:56,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:51:56,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:01,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:03,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:52:03,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:52:03,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:52:03,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:52:06,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:06,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:11,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:16,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:17,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:18,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:52:18,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:52:18,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:52:21,070 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:52:22,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:26,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:27,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:32,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:33,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:52:33,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:52:33,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:52:34,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:38,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:43,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:44,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:48,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:52:48,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:52:48,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:51,071 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:52:54,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:52:55,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:52:59,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:03,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:53:03,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:53:04,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:05,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:53:09,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:14,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:15,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:53:18,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:53:18,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:53:19,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:21,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:53:23,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:53:25,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:30,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:33,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:53:33,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:53:33,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:53:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:40,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:43,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:53:45,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:48,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:53:48,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:53:48,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:53:48,331 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:53:51,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:53:51,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:53:53,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:53:56,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:02,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:02,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:03,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:54:03,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:54:03,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:54:07,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:12,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:13,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:18,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:19,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:54:19,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:54:19,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:54:19,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:54:21,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:54:22,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:24,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:29,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:30,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:34,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:34,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:54:34,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:54:34,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:54:39,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:40,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:45,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:49,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:54:49,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:54:49,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:54:49,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:54:49,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:54:49,926 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:54:49,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:54:49,929 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:54:50,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:54:50,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:50,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:54:51,074 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:54:51,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:54:56,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:00,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:55:01,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:04,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:55:04,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:55:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:11,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:55:13,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:18,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:19,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:55:19,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:55:21,074 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:55:21,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:55:24,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:29,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:55:29,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:34,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:55:34,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:55:34,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:39,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:55:40,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:45,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:49,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:55:49,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:55:49,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:55:49,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:55:49,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:55:50,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:51,075 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:55:56,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:55:59,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:56:01,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:04,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:56:04,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:56:04,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:56:06,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:09,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:56:12,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:17,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:19,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:56:19,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:56:19,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:56:19,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:56:21,075 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:56:23,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:28,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:29,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:56:33,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:36,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:56:36,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:56:36,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:56:38,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:39,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:56:43,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:49,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:49,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:56:51,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:56:51,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:56:51,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:56:51,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:56:54,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:56:58,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:00,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:05,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:06,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:57:06,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:57:06,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:57:06,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:57:06,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:10,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:15,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:17,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:21,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:21,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:57:21,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:57:21,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:57:21,252 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:57:26,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:27,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:31,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:36,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:57:36,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:57:36,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:57:37,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:37,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:42,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:47,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:48,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:51,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:57:51,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:57:51,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:57:51,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:57:53,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:57:57,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:57:59,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:04,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:05,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 03:58:05,155 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 03:58:05,156 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 03:58:05,159 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 03:58:05,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 03:58:05,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:06,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:58:06,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:58:06,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:58:07,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:09,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:14,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:17,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:19,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:21,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:58:21,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:58:21,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:58:21,053 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:58:21,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:58:24,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:25,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:29,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:35,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:35,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:36,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:58:36,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:58:36,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:58:41,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:45,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:46,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:51,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:58:51,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:58:51,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:58:51,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:58:51,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:58:52,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:58:54,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:58:57,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:03,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:59:03,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:06,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:59:06,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:59:06,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:59:08,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:13,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:59:14,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:19,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:21,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:59:21,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:59:21,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:59:21,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:59:21,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 03:59:23,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:59:25,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:30,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:33,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:59:35,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:36,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:59:36,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:59:40,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:43,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:59:46,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:51,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 03:59:51,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 03:59:51,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 03:59:52,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 03:59:53,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 03:59:57,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:02,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:03,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:00:06,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:00:06,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:00:08,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:11,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:00:13,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:18,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:21,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:00:21,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:00:21,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:00:21,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:00:24,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:29,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:34,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:36,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:00:36,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:00:37,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:00:39,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:44,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:47,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:00:49,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:51,081 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:00:51,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:00:51,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:00:55,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:00:59,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:00,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:05,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:06,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:01:06,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:01:07,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:11,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:16,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:20,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:21,081 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:01:21,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:21,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:01:21,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:01:26,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:26,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:01:26,626 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:01:26,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:01:26,630 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:01:27,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:01:27,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:28,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:31,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:36,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:01:36,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:01:37,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:37,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:42,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:47,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:47,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:51,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:01:51,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:01:51,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:01:53,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:01:57,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:01:58,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:03,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:06,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:02:06,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:02:07,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:02:08,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:13,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:17,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:02:19,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:21,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:02:21,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:02:21,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:02:25,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:27,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:02:30,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:35,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:35,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:02:36,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:02:36,830 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:02:41,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:45,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:02:46,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:51,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:02:52,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:02:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:02:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:02:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:02:52,741 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:02:55,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:02:57,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:02,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:05,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:03:07,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:03:07,739 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:03:07,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:12,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:15,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:03:18,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:21,085 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:03:22,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:03:22,739 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:03:23,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:24,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:03:28,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:34,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:03:34,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:37,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:03:37,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:03:37,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:03:37,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:03:39,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:44,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:03:45,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:50,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:03:51,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:03:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:03:52,739 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:03:54,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:03:55,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:01,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:01,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:06,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:09,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:04:09,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:04:09,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:04:09,262 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:04:11,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:12,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:17,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:21,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:04:21,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:23,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:24,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:04:24,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:04:24,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:04:24,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:04:28,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:31,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:34,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:39,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:39,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:04:39,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:04:39,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:04:39,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:04:39,481 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:04:39,481 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:04:39,484 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:04:39,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:04:40,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:41,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:44,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:50,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:51,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:04:51,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:04:54,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:04:54,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:04:54,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:04:55,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:04:59,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:05:00,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:05,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:09,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:05:09,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:05:09,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:05:09,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:05:11,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:16,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:19,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:05:21,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:05:22,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:24,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:05:24,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:05:24,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:05:24,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:05:28,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:29,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:05:33,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:38,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:05:38,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:39,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:05:39,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:05:39,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:05:44,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:48,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:05:49,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:51,088 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:05:54,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:05:54,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:05:54,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:05:54,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:05:54,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:05:58,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:00,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:06,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:08,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:09,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:06:09,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:06:09,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:06:12,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:16,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:17,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:21,089 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:06:23,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:24,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:06:24,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:06:24,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:06:26,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:28,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:34,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:36,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:39,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:39,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:06:39,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:06:39,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:06:45,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:45,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:50,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:06:51,091 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:06:53,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:06:54,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:06:54,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:06:54,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:06:55,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:00,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:03,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:06,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:10,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:07:10,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:07:10,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:07:10,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:07:11,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:13,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:16,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:21,091 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:07:22,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:23,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:25,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:07:25,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:07:27,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:33,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:33,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:38,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:40,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:07:40,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:07:40,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:07:40,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:07:43,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:43,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:48,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:51,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:07:51,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:07:51,111 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:07:51,112 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:07:51,116 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:07:51,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:07:51,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:52,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:07:54,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:07:55,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:07:55,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:07:55,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:07:55,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:07:59,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:02,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:08:05,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:10,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:08:10,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:08:10,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:08:10,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:12,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:08:15,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:21,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:08:21,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:22,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:08:25,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:08:25,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:08:26,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:32,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:32,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:08:37,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:40,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:08:40,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:08:42,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:08:42,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:47,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:50,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:08:51,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:08:53,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:08:57,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:08:57,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:08:57,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:08:57,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:08:58,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:00,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:03,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:08,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:10,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:12,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:09:12,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:09:12,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:09:14,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:19,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:20,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:21,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:09:25,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:27,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:09:27,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:09:28,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:30,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:35,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:37,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:41,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:42,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:09:42,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:09:46,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:47,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:51,094 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:09:52,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:09:55,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:09:57,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:09:57,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:09:57,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:02,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:05,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:10:08,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:13,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:13,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:10:13,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:10:13,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:10:13,354 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:10:15,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:10:18,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:21,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:10:23,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:26,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:10:28,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:10:28,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:10:28,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:10:28,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:10:29,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:34,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:10:35,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:40,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:10:43,356 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:10:44,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:10:45,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:50,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:51,098 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:10:55,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:10:56,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:10:58,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:10:58,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:10:58,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:10:58,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:11:00,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:03,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:11:03,435 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:11:03,436 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:11:03,438 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:11:04,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:11:04,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:05,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:06,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:11,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:13,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:11:13,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:11:13,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:11:14,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:16,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:21,098 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:11:22,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:24,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:27,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:28,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:11:28,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:11:28,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:11:33,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:34,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:38,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:43,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:11:43,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:11:43,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:11:43,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:11:43,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:44,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:48,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:51,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:11:52,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:11:54,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:59,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:11:59,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:11:59,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:11:59,809 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:12:02,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:12:04,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:09,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:12,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:12:14,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:12:14,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:12:14,809 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:12:14,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:19,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:21,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:12:22,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:12:25,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:29,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:12:29,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:12:29,809 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:12:31,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:32,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:12:36,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:39,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:12:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:44,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:12:44,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:12:44,809 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:12:47,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:49,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:12:51,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:12:52,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:57,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:12:59,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:12:59,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:12:59,809 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:13:00,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:13:03,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:08,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:10,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:13:13,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:14,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:13:14,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:13:14,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:13:14,814 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:13:18,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:13:18,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:21,100 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:13:24,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:28,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:13:29,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:13:29,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:13:29,810 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:13:30,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:35,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:38,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:13:40,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:45,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:13:45,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:13:45,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:13:45,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:13:46,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:48,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:13:51,101 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:13:52,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:57,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:13:58,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:14:00,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:14:00,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:14:00,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:14:00,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:14:03,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:08,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:14:08,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:13,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:14:15,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:14:15,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:14:15,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:14:15,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:14:15,347 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:14:15,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:14:15,350 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:14:15,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:14:16,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:14:19,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:21,101 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:14:24,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:26,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:14:29,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:30,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:14:30,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:14:30,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:14:35,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:40,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:14:41,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:45,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:14:45,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:14:45,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:14:46,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:50,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:14:51,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:14:52,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:14:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:00,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:00,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:15:00,048 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:15:00,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:02,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:07,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:11,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:12,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:15:16,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:15:18,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:19,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:21,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:15:24,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:28,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:30,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:31,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:31,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:31,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:15:31,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:15:35,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:38,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:41,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:46,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:46,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:46,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:15:46,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:15:46,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:15:48,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:51,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:15:52,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:15:56,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:15:57,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:01,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:16:01,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:16:01,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:16:03,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:06,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:16:09,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:14,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:16:16,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:16:16,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:16:16,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:16:19,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:21,103 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:16:25,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:26,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:16:30,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:31,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:16:31,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:16:31,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:16:35,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:36,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:16:40,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:46,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:46,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:16:46,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:16:46,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:16:46,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:16:51,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:16:52,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:16:54,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:16:57,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:01,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:17:01,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:17:01,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:17:01,849 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:17:02,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:04,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:17:07,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:13,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:14,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:17:16,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:17:16,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:17:16,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:17:19,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:21,107 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:17:24,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:25,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:17:29,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:31,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:17:31,538 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:17:31,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:17:31,542 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:17:31,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:17:31,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:17:31,844 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:17:32,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:17:33,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:17:35,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:40,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:43,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:17:45,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:46,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:17:46,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:17:46,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:17:51,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:17:51,107 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:17:53,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:17:56,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:01,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:01,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:18:01,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:18:01,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:18:03,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:18:07,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:11,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:18:12,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:16,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:18:16,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:18:16,849 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:18:18,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:20,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:18:21,108 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:18:23,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:28,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:30,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:18:31,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:18:31,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:18:31,846 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:18:34,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:38,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:18:39,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:44,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:47,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:18:47,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:18:47,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:18:47,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:18:49,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:18:50,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:18:51,108 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:18:55,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:00,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:00,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:02,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:19:02,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:19:02,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:19:02,243 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:19:05,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:08,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:10,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:15,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:17,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:19:17,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:19:17,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:19:17,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:19:18,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:21,109 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:19:21,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:26,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:28,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:31,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:32,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:19:32,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:19:32,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:19:37,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:38,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:42,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:47,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:47,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:19:47,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:19:47,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:19:47,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:51,109 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:19:53,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:19:57,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:19:58,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:03,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:03,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:20:03,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:20:03,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:20:03,782 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:20:07,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:20:08,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:13,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:17,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:20:18,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:20:18,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:20:18,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:20:19,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:21,111 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:20:24,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:25,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:20:29,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:33,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:20:33,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:20:34,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:35,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:20:40,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:43,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:20:43,634 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:20:43,634 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:20:43,636 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:20:44,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:20:45,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:20:45,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:48,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:20:48,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:20:48,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:20:48,780 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:20:50,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:20:51,113 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:20:55,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:20:56,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:01,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:03,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:21:03,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:21:03,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:21:03,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:21:07,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:12,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:21:12,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:17,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:20,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:21:20,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:21:20,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:21:20,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:21:21,113 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:21:22,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:21:23,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:28,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:32,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:21:33,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:35,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:21:35,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:21:35,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:21:39,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:40,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:21:45,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:50,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:21:50,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:21:50,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:21:50,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:21:50,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:21:51,113 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:21:56,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:00,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:22:01,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:05,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:22:05,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:22:05,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:22:07,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:11,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:22:13,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:18,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:20,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:22:20,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:22:20,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:22:20,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:22:21,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:22:21,113 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:22:24,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:29,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:31,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:22:34,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:35,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:22:35,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:22:35,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:22:39,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:22:40,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:45,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:49,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:22:50,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:22:50,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:22:50,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:22:50,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:51,114 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:22:56,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:22:59,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:01,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:05,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:23:05,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:23:05,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:23:07,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:07,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:12,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:17,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:18,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:20,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:23:20,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:23:20,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:23:21,115 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:23:23,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:27,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:28,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:33,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:35,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:23:35,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:23:35,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:23:37,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:39,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:45,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:45,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:50,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:23:50,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:23:50,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:23:50,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:23:51,116 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:23:53,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:23:53,826 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:23:53,826 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:23:53,829 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:23:54,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:23:55,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:23:55,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:00,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:05,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:24:05,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:24:05,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:24:05,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:24:06,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:11,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:14,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:24:17,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:21,129 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:24:21,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:24:21,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:24:21,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:24:22,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:22,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:24:27,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:33,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:24:33,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:36,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:24:36,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:24:36,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:24:36,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:24:38,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:43,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:24:44,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:49,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:51,117 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:24:51,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:24:51,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:24:51,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:24:53,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:24:54,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:24:59,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:03,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:25:05,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:06,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:25:06,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:25:06,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:25:10,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:11,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:25:15,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:21,118 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:25:21,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:21,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:25:21,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:25:21,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:25:21,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:25:26,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:31,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:25:32,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:37,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:37,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:25:37,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:25:37,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:25:41,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:25:43,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:48,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:51,118 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:25:51,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:25:52,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:25:52,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:25:52,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:25:53,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:58,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:25:59,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:03,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:07,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:26:07,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:26:07,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:26:09,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:09,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:14,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:19,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:19,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:21,119 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:26:22,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:26:22,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:26:22,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:26:25,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:29,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:30,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:35,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:37,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:26:37,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:26:37,954 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:26:37,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:41,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:46,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:48,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:51,119 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:26:52,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:26:54,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:26:54,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:26:54,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:26:54,470 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:26:57,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:26:57,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:02,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:05,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:27:05,189 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:27:05,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:27:05,192 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:27:06,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:27:07,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:08,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:08,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:09,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:27:09,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:27:09,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:27:13,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:15,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:19,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:21,121 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:27:24,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:27:24,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:27:24,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:27:24,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:25,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:29,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:35,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:35,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:39,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:27:39,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:27:39,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:27:40,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:45,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:46,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:51,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:27:52,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:27:53,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:27:54,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:27:54,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:27:54,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:27:57,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:03,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:03,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:28:08,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:10,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:28:10,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:28:10,526 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:28:13,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:28:13,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:18,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:21,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:28:23,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:28:24,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:25,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:28:25,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:28:29,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:33,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:28:35,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:40,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:40,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:28:40,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:28:40,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:28:41,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:28:45,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:50,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:28:51,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:28:55,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:28:55,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:28:55,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:28:56,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:01,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:05,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:29:07,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:10,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:29:10,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:29:13,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:16,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:29:19,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:21,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:29:25,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:25,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:29:25,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:29:26,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:29:30,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:35,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:36,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:29:40,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:29:40,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:29:40,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:29:40,530 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:29:41,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:46,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:29:47,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:51,125 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:29:52,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:29:53,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:29:55,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:29:55,526 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:29:58,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:04,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:05,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:09,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:10,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:30:10,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:30:10,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:30:15,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:15,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:20,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:21,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:30:23,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:30:23,103 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:30:23,104 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:30:23,106 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:30:23,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:30:23,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:24,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:25,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:30:25,526 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:30:25,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:30,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:33,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:35,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:40,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:41,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:30:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:30:41,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:30:41,169 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:30:43,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:46,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:51,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:30:51,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:30:53,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:30:56,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:30:56,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:30:56,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:30:57,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:02,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:03,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:31:07,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:11,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:31:11,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:31:11,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:31:11,166 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:31:13,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:13,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:31:18,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:21,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:31:21,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:31:24,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:26,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:31:26,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:31:26,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:31:29,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:32,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:31:35,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:40,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:31:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:31:41,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:31:42,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:31:46,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:51,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:31:52,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:52,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:31:57,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:31:57,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:31:57,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:31:57,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:31:57,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:32:00,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:32:02,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:07,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:10,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:32:12,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:32:12,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:32:12,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:32:12,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:18,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:19,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:32:21,128 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:32:24,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:27,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:32:27,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:32:27,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:32:27,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:32:29,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:32:29,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:34,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:37,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:32:40,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:42,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:32:42,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:32:42,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:32:45,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:47,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:32:50,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:51,128 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:32:56,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:32:57,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:32:57,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:32:57,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:01,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:05,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:07,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:12,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:13,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:33:13,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:33:13,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:33:13,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:33:15,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:18,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:21,130 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:33:24,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:25,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:28,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:33:28,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:33:28,344 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:33:29,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:33,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:33:33,480 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:33:33,480 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:33:33,483 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:33:33,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:33:33,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:35,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:36,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:40,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:43,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:33:43,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:33:43,344 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:33:44,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:45,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:50,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:51,132 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:33:54,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:33:55,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:33:58,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:33:58,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:33:58,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:34:01,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:04,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:34:07,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:12,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:13,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:34:13,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:34:13,345 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:34:14,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:34:17,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:21,132 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:34:22,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:34:23,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:28,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:29,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:34:29,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:34:29,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:34:29,784 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:34:32,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:34:33,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:38,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:42,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:34:43,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:44,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:34:44,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:34:44,782 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:34:48,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:50,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:34:51,132 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:34:54,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:59,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:34:59,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:34:59,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:34:59,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:34:59,782 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:35:00,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:35:05,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:10,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:35:11,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:14,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:35:14,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:35:14,782 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:35:16,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:19,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:35:21,133 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:35:21,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:26,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:29,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:35:29,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:35:29,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:35:29,783 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:35:32,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:37,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:39,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:35:42,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:46,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:35:46,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:35:46,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:35:46,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:35:47,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:35:48,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:51,133 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:35:53,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:58,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:35:59,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:01,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:36:01,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:36:01,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:36:01,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:36:04,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:08,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:10,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:15,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:16,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:36:16,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:36:16,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:36:16,005 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:36:18,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:20,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:21,134 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:36:26,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:28,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:31,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:36:31,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:36:31,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:36:31,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:36,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:36,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:41,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:45,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:36:45,455 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:36:45,456 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:36:45,459 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:36:46,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:36:46,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:36:46,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:36:46,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:36:46,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:51,134 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:36:53,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:36:56,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:36:58,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:01,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:37:01,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:37:01,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:37:01,006 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:37:04,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:06,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:37:09,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:14,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:14,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:37:16,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:37:16,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:37:16,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:37:20,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:21,136 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:37:24,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:37:25,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:30,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:31,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:37:31,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:37:31,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:37:34,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:37:36,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:41,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:42,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:37:46,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:37:46,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:37:46,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:37:46,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:51,136 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:37:52,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:37:52,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:37:57,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:01,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:38:01,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:38:01,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:38:02,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:02,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:38:07,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:12,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:38:13,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:16,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:38:16,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:38:16,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:38:18,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:19,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:38:21,136 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:38:24,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:29,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:29,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:38:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:38:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:38:31,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:38:34,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:39,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:40,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:38:44,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:46,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:38:46,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:38:46,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:38:50,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:38:51,137 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:38:56,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:00,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:39:01,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:39:01,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:39:01,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:39:01,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:06,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:08,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:39:11,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:16,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:39:16,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:39:16,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:39:17,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:18,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:39:21,137 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:39:23,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:28,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:39:28,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:39:31,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:39:31,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:39:33,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:36,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:39:38,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:43,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:46,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:39:46,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:39:46,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:39:46,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:39:49,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:51,138 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:39:55,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:39:56,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:00,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:01,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:40:01,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:40:01,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:40:04,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:40:04,362 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:40:04,363 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:40:04,366 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:40:04,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:40:05,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:06,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:10,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:14,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:15,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:16,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:40:16,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:40:16,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:40:21,140 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:40:21,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:24,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:26,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:31,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:31,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:40:31,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:40:31,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:40:31,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:40:34,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:36,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:41,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:44,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:46,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:40:46,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:40:46,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:40:47,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:51,140 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:40:52,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:40:55,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:40:57,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:01,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:41:01,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:41:01,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:41:01,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:41:03,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:05,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:41:08,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:13,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:41:13,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:16,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:41:16,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:41:16,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:41:18,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:21,141 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:41:22,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:41:24,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:29,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:31,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:41:31,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:41:31,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:41:32,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:41:34,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:39,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:42,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:41:45,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:46,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:41:46,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:41:46,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:41:50,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:41:50,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:41:51,141 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:41:56,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:00,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:01,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:01,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:42:01,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:42:01,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:06,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:10,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:11,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:16,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:16,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:16,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:16,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:42:16,810 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:42:18,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:21,142 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:42:22,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:27,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:28,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:31,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:42:31,807 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:42:33,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:38,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:40,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:43,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:42:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:42:46,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:42:48,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:48,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:51,142 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:42:54,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:42:58,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:42:59,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:43:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:43:01,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:43:05,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:08,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:43:10,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:15,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:16,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:43:16,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:43:16,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:43:20,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:43:20,486 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:43:20,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:43:20,489 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:43:21,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:43:21,142 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:43:21,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:23,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:43:26,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:31,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:43:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:43:31,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:43:31,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:43:31,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:36,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:41,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:43:42,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:47,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:48,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:43:48,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:43:48,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:43:48,135 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:43:51,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:43:51,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:43:53,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:43:58,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:00,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:03,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:44:03,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:44:03,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:44:03,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:08,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:08,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:13,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:18,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:44:18,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:44:18,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:44:18,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:19,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:21,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:44:25,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:28,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:30,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:44:33,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:44:33,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:44:36,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:38,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:42,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:46,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:47,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:48,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:44:48,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:44:48,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:44:48,136 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:44:51,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:44:53,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:44:56,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:44:58,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:03,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:45:04,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:45:04,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:45:06,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:45:09,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:14,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:16,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:45:19,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:45:19,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:45:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:21,145 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:45:24,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:26,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:45:29,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:34,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:45:34,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:45:34,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:45:35,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:40,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:45,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:49,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:45:49,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:45:49,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:45:50,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:45:51,145 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:45:53,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:45:56,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:02,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:03,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:46:04,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:46:04,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:46:04,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:46:07,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:13,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:46:13,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:18,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:19,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:46:19,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:46:19,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:46:21,146 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:46:23,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:46:24,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:29,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:33,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:46:34,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:46:34,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:46:34,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:46:34,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:39,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:43,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:46:45,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:49,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:46:49,563 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:46:49,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:46:49,566 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:46:49,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:46:49,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:46:49,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:46:50,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:46:50,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:46:51,146 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:46:51,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:46:56,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:00,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:47:01,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:04,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:47:04,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:47:04,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:47:06,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:10,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:47:12,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:17,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:18,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:47:19,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:47:19,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:47:19,729 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:47:21,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:47:23,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:28,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:28,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:47:33,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:35,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:47:35,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:47:35,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:47:35,814 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:47:38,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:47:38,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:43,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:46,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:47:49,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:50,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:47:50,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:47:50,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:47:51,147 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:47:55,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:47:57,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:00,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:05,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:48:05,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:48:06,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:07,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:11,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:16,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:17,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:20,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:48:20,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:48:21,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:48:22,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:27,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:27,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:32,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:35,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:35,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:48:35,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:48:38,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:44,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:45,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:49,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:51,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:48:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:48:51,096 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:48:51,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:48:53,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:48:54,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:48:59,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:03,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:49:04,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:06,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:49:06,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:49:06,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:49:10,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:13,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:49:15,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:20,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:21,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:49:21,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:49:21,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:49:21,255 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:49:23,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:49:26,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:31,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:33,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:49:36,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:49:36,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:49:36,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:49:37,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:40,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:49:42,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:47,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:50,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:49:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:49:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:49:51,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:49:51,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:49:53,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:58,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:49:58,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:49:58,623 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:49:58,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:49:58,627 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:49:58,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:49:58,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:00,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:03,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:06,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:50:06,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:50:06,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:50:09,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:09,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:14,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:19,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:19,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:21,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:50:21,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:50:21,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:50:21,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:50:21,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:50:25,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:29,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:30,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:35,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:36,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:50:36,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:50:36,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:50:36,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:50:39,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:41,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:46,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:49,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:50:51,152 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:50:51,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:51,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:50:51,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:50:56,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:50:59,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:01,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:06,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:51:06,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:51:06,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:51:06,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:51:07,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:07,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:12,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:17,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:17,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:21,152 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:51:21,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:51:21,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:51:23,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:27,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:28,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:33,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:36,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:51:36,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:51:37,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:39,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:44,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:47,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:49,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:51,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:51:51,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:51:51,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:51:51,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:51:54,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:51:57,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:51:59,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:05,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:05,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:52:06,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:52:06,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:52:10,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:14,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:52:15,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:20,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:21,154 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:52:22,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:52:22,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:52:22,249 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:52:25,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:52:26,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:31,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:35,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:52:37,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:37,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:52:37,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:52:37,247 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:52:42,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:45,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:52:47,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:51,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:52:52,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:52:52,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:52:52,247 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:52:53,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:52:53,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:52:58,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:03,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:03,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:07,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:53:07,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:53:07,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:53:09,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:11,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:53:11,028 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:53:11,028 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:53:11,031 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:53:11,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:53:11,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:13,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:15,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:20,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:21,155 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:53:21,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:22,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:53:22,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:53:22,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:53:25,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:30,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:31,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:35,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:38,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:53:38,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:53:38,235 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:53:41,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:41,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:46,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:51,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:53:51,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:53:52,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:53:53,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:53:53,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:53:53,235 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:53:57,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:01,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:54:03,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:08,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:08,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:54:08,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:54:08,236 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:54:09,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:54:13,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:19,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:54:21,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:54:23,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:54:23,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:54:23,235 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:54:24,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:29,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:54:30,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:35,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:38,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:54:38,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:54:38,235 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:54:40,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:54:40,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:45,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:48,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:54:51,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:54:51,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:54,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:54:54,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:54:54,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:54:54,592 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:54:56,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:54:58,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:01,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:07,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:07,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:09,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:55:09,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:55:09,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:55:12,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:15,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:18,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:21,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:55:23,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:24,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:55:24,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:55:25,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:29,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:35,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:35,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:39,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:55:39,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:55:40,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:45,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:46,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:51,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:55:52,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:55:53,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:55:54,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:55:54,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:55:57,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:03,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:03,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:56:08,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:10,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:56:10,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:56:10,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:56:10,488 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:56:13,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:56:13,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:18,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:21,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:56:21,034 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:56:21,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:56:21,037 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:56:21,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:56:21,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:56:23,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:56:24,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:25,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:56:25,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:56:25,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:56:29,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:33,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:56:35,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:40,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:40,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:56:40,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:56:40,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:56:40,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:56:41,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:56:45,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:51,162 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:56:51,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:56:51,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:56:55,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:56:55,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:56:55,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:56:56,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:01,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:57:02,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:07,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:10,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:57:10,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:57:10,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:57:12,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:57:12,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:17,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:21,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:57:22,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:57:23,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:25,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:57:25,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:57:25,487 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:57:28,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:34,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:36,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:57:39,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:41,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:57:41,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:57:41,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:57:41,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:57:44,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:57:44,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:49,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:51,163 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:57:54,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:57:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:57:56,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:57:56,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:57:56,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:58:00,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:03,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:58:06,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:11,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:58:11,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:58:11,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:58:11,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:13,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:58:17,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:21,164 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:58:22,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:23,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:58:26,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:58:26,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:58:26,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:58:27,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:33,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:33,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:58:38,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:41,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:58:41,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:58:41,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:58:43,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:58:43,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:48,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:51,164 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:58:53,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:58:54,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:58:56,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:58:56,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:58:56,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:58:59,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:01,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:59:05,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:10,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:11,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:59:11,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:59:11,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:59:11,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:59:15,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:21,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:59:21,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:22,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:59:26,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:27,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:59:27,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:59:27,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:59:30,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:59:31,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:36,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:38,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 04:59:38,351 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 04:59:38,352 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 04:59:38,354 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 04:59:39,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 04:59:40,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:59:42,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:42,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:59:42,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:59:42,906 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:59:47,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:50,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 04:59:51,166 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 04:59:53,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 04:59:57,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 04:59:57,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 04:59:57,906 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 04:59:58,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:00,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:03,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:09,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:10,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:12,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:00:12,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:00:12,906 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:00:14,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:18,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:19,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:21,167 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:00:25,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:27,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:00:27,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:00:27,906 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:00:28,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:31,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:36,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:38,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:41,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:43,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:00:43,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:00:43,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:00:43,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:00:45,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:47,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:51,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:00:52,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:55,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:00:57,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:00:58,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:00:58,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:00:58,910 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:01:03,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:05,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:01:08,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:13,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:13,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:01:13,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:01:13,889 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:01:15,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:01:19,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:21,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:01:24,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:25,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:01:28,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:01:28,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:01:28,889 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:01:30,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:33,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:01:35,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:40,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:43,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:01:43,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:01:43,889 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:01:43,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:01:46,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:51,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:01:51,171 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:01:54,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:01:56,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:00,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:02:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:02:00,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:02:00,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:02:01,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:04,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:02:06,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:11,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:12,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:02:15,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:02:15,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:02:15,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:02:17,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:21,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:02:22,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:02:23,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:28,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:02:30,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:02:30,334 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:02:32,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:02:33,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:38,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:40,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:02:44,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:45,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:02:45,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:02:45,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:02:48,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:02:48,411 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:02:48,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:02:48,414 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:02:48,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:02:49,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:02:50,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:02:51,171 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:02:55,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:03:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:03:00,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:03:00,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:00,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:05,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:08,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:10,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:15,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:03:15,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:03:15,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:03:15,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:03:16,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:18,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:21,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:03:22,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:27,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:28,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:03:30,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:03:30,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:03:33,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:37,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:38,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:43,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:45,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:03:45,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:03:45,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:03:45,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:51,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:03:54,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:03:56,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:03:59,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:04:00,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:04:00,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:04:04,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:06,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:04:09,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:14,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:04:15,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:04:15,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:04:15,353 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:04:15,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:20,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:21,173 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:04:24,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:04:25,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:30,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:04:30,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:04:30,356 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:04:31,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:34,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:04:37,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:42,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:44,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:04:45,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:04:45,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:04:45,354 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:04:47,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:51,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:04:52,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:04:54,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:04:57,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:01,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:05:01,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:05:01,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:05:01,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:05:03,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:04,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:05:08,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:13,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:14,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:05:16,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:05:16,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:05:16,055 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:05:19,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:21,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:05:25,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:30,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:31,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:05:31,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:05:31,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:05:35,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:40,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:45,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:46,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:05:46,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:05:46,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:05:51,176 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:05:51,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:56,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:05:58,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:01,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:06:01,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:06:01,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:06:01,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:06,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:08,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:11,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:16,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:06:16,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:06:16,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:06:16,059 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:06:17,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:18,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:21,176 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:06:23,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:27,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:28,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:31,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:06:31,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:06:31,056 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:06:34,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:35,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:06:35,627 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:06:35,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:06:35,630 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:06:36,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:06:36,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:38,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:39,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:44,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:46,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:06:46,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:06:46,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:06:46,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:06:50,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:51,178 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:06:55,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:06:56,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:07:00,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:01,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:07:01,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:07:01,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:07:01,060 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:07:05,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:06,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:07:10,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:16,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:07:16,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:07:16,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:07:16,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:16,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:07:21,180 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:07:22,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:26,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:07:27,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:31,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:07:31,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:07:33,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:34,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:07:38,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:43,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:46,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:07:46,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:07:48,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:51,180 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:07:54,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:07:59,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:01,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:08:01,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:08:04,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:08,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:08:09,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:14,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:16,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:08:16,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:08:18,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:08:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:21,181 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:08:26,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:28,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:08:31,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:31,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:08:31,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:08:36,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:38,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:08:41,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:46,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:08:46,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:08:46,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:08:47,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:51,181 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:08:53,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:08:56,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:08:58,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:03,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:09:03,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:09:03,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:09:04,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:07,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:09:09,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:14,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:15,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:09:18,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:09:18,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:09:18,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:09:20,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:21,183 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:09:25,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:09:25,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:30,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:33,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:09:33,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:09:33,133 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:09:35,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:36,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:09:40,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:44,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:09:46,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:48,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:09:48,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:09:48,132 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:09:51,184 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:09:52,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:09:54,371 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:09:57,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:03,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:10:03,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:10:03,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:10:03,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:04,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:10:08,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:11,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:10:11,551 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:10:11,552 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:10:11,555 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:10:12,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:10:12,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:10:13,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:18,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:10:18,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:10:18,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:10:19,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:21,185 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:10:22,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:10:24,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:29,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:32,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:10:33,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:10:33,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:10:33,131 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:10:34,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:39,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:42,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:10:45,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:48,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:10:48,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:10:48,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:10:48,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:10:50,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:10:50,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:10:51,185 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:10:56,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:02,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:02,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:11:03,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:11:03,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:11:03,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:11:03,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:11:07,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:10,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:11:12,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:17,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:18,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:11:18,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:11:20,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:11:21,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:11:23,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:28,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:31,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:11:33,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:11:33,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:11:33,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:39,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:41,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:11:44,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:48,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:11:48,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:11:49,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:51,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:11:51,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:11:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:11:59,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:12:00,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:03,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:12:03,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:12:05,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:09,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:12:11,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:16,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:18,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:12:18,678 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:12:19,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:12:21,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:12:21,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:26,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:28,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:12:31,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:12:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:12:33,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:12:36,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:42,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:42,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:12:47,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:50,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:12:50,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:12:50,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:12:50,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:12:51,187 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:12:52,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:12:52,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:12:57,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:02,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:02,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:13:05,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:13:05,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:13:05,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:13:05,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:13:07,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:12,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:13:12,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:17,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:20,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:13:20,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:13:20,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:13:20,148 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:13:21,188 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:13:22,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:13:23,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:28,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:30,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:13:30,103 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:13:30,104 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:13:30,106 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:13:30,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:13:30,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:13:34,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:35,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:13:35,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:13:35,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:13:39,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:40,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:13:44,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:50,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:13:50,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:13:50,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:13:50,147 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:13:51,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:13:51,188 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:13:55,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:00,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:01,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:14:05,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:14:05,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:14:05,150 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:14:06,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:11,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:14:12,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:17,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:19,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:14:20,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:14:20,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:14:20,147 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:14:21,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:14:23,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:28,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:29,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:14:33,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:35,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:14:35,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:14:35,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:14:35,941 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:14:39,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:39,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:14:44,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:49,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:49,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:14:50,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:14:50,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:14:50,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:14:50,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:14:51,189 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:14:55,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:14:57,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:01,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:05,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:05,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:15:05,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:15:06,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:07,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:11,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:16,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:16,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:20,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:20,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:20,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:15:20,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:15:21,190 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:15:22,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:26,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:27,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:32,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:35,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:35,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:35,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:15:35,941 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:15:36,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:38,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:43,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:46,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:48,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:50,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:50,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:15:50,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:15:50,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:15:51,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:15:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:15:54,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:15:59,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:04,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:05,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:16:05,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:16:05,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:16:05,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:16:05,940 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:16:10,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:15,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:16:15,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:20,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:21,192 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:16:22,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:16:22,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:16:22,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:16:22,213 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:16:25,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:16:26,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:31,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:35,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:16:36,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:37,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:16:37,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:16:37,211 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:16:37,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:16:42,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:42,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:16:42,349 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:16:42,350 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:16:42,353 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:16:43,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:16:43,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:16:47,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:51,193 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:16:52,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:16:52,211 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:16:52,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:16:53,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:16:57,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:02,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:03,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:17:07,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:17:07,211 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:17:08,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:13,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:17:13,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:18,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:21,194 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:17:21,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:17:22,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:17:22,211 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:17:24,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:29,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:31,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:17:34,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:37,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:17:37,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:17:37,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:17:37,214 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:17:40,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:40,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:17:45,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:50,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:17:51,194 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:17:51,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:17:52,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:17:52,212 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:17:56,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:00,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:18:01,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:06,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:08,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:18:08,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:18:08,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:18:11,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:18:12,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:17,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:21,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:18:21,194 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:18:22,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:23,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:18:23,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:18:27,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:29,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:18:32,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:38,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:18:38,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:18:38,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:39,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:18:43,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:48,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:49,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:18:51,195 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:18:53,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:18:53,144 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:18:54,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:18:57,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:00,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:05,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:07,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:08,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:19:08,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:19:08,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:19:08,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:19:10,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:15,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:17,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:20,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:21,198 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:19:23,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:19:23,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:19:23,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:19:23,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:19:25,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:26,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:31,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:35,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:37,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:38,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:19:38,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:19:42,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:45,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:48,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:51,198 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:19:52,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:19:52,473 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:19:52,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:19:52,475 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:19:52,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:19:53,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:19:53,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:19:53,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:19:53,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:19:58,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:03,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:03,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:20:08,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:20:08,185 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:20:09,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:13,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:20:15,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:20,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:21,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:20:23,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:20:23,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:20:23,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:20:25,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:30,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:34,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:20:35,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:39,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:20:39,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:20:39,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:20:39,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:20:41,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:20:41,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:46,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:51,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:20:51,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:20:52,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:20:54,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:20:54,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:20:54,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:20:54,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:20:57,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:01,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:21:02,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:07,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:09,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:21:09,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:21:09,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:21:11,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:21:13,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:18,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:19,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:21:21,200 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:21:24,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:21:24,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:21:24,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:21:24,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:29,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:29,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:21:34,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:39,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:21:39,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:21:39,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:21:39,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:39,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:21:44,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:49,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:21:50,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:51,212 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:21:54,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:21:54,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:21:54,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:21:55,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:21:59,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:22:00,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:05,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:07,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:22:09,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:22:09,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:22:09,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:22:11,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:17,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:17,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:22:21,200 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:22:22,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:24,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:22:24,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:22:24,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:22:24,991 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:22:27,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:27,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:22:32,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:37,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:22:37,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:39,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:22:39,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:22:43,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:47,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:22:48,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:51,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:22:54,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:22:54,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:22:54,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:22:54,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:22:54,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:22:56,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:00,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:04,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:23:04,829 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:23:04,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:23:04,832 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:23:05,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:23:05,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:06,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:07,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:09,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:23:09,987 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:23:11,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:16,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:16,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:21,202 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:23:22,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:24,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:23:24,988 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:23:26,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:28,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:34,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:35,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:39,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:41,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:23:41,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:23:41,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:23:41,401 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:23:43,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:44,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:49,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:51,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:23:53,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:23:55,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:23:56,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:23:56,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:23:56,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:23:56,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:24:00,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:03,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:24:06,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:11,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:11,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:24:11,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:24:11,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:24:11,400 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:24:13,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:24:16,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:21,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:24:21,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:24:22,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:26,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:24:26,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:24:26,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:24:27,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:31,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:24:32,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:37,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:41,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:24:41,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:24:41,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:24:41,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:24:43,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:48,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:51,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:24:51,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:24:54,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:24:56,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:24:56,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:24:56,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:24:59,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:02,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:25:05,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:10,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:11,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:25:11,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:25:11,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:25:12,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:25:16,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:21,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:25:22,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:25:22,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:27,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:28,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:25:28,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:25:28,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:25:28,038 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:25:32,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:25:33,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:38,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:40,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:25:43,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:25:43,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:25:43,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:25:43,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:48,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:50,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:25:51,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:25:54,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:25:58,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:25:58,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:25:58,038 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:25:59,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:00,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:04,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:09,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:09,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:13,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:26:13,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:26:13,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:26:15,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:17,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:26:17,050 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:26:17,050 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:26:17,052 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:26:17,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:26:17,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:19,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:21,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:21,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:26:26,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:28,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:26:28,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:26:28,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:26:29,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:31,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:36,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:37,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:42,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:43,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:26:43,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:26:43,037 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:26:47,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:47,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:26:51,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:26:53,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:58,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:26:59,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:26:59,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:26:59,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:26:59,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:27:01,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:27:03,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:08,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:11,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:27:14,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:14,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:27:14,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:27:14,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:27:19,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:21,207 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:27:22,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:27:25,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:29,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:27:29,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:27:29,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:27:29,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:27:30,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:32,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:27:35,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:40,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:27:40,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:44,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:27:44,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:27:46,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:50,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:27:51,207 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:27:52,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:57,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:27:59,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:27:59,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:27:59,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:28:00,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:28:02,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:07,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:10,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:28:12,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:14,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:28:14,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:28:14,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:28:18,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:20,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:28:21,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:28:24,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:29,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:28:29,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:28:29,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:28:29,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:30,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:28:34,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:38,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:28:39,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:44,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:45,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:28:45,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:28:45,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:28:45,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:28:48,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:28:50,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:51,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:28:56,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:28:57,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:00,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:00,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:00,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:29:00,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:29:01,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:06,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:07,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:11,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:15,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:15,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:15,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:29:15,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:29:15,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:17,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:21,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:29:23,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:25,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:28,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:30,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:30,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:30,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:29:30,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:29:33,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:33,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:29:33,640 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:29:33,641 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:29:33,643 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:29:33,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:29:35,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:38,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:43,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:44,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:45,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:29:45,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:29:45,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:29:49,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:29:51,223 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:29:54,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:29:55,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:00,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:01,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:01,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:01,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:30:01,793 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:30:04,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:30:06,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:11,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:14,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:30:16,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:16,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:16,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:16,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:30:16,791 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:30:21,212 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:30:22,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:24,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:30:27,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:31,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:31,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:31,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:30:31,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:30:33,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:34,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:30:38,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:43,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:44,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:30:46,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:30:46,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:30:46,791 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:30:49,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:51,214 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:30:54,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:30:54,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:30:59,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:01,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:31:01,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:31:01,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:31:01,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:31:04,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:31:05,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:10,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:12,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:31:15,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:16,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:31:16,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:31:16,791 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:31:21,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:21,214 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:31:22,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:31:26,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:31,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:31,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:31:31,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:31:31,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:31:32,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:31:37,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:42,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:42,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:31:46,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:31:46,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:31:46,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:31:48,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:51,215 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:31:51,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:31:53,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:31:58,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:01,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:32:01,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:32:01,792 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:32:02,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:32:04,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:09,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:10,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:32:14,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:17,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:32:17,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:32:17,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:32:17,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:32:19,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:20,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:32:21,215 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:32:25,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:30,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:32:31,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:32,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:32:32,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:32:32,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:32:32,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:32:36,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:38,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:32:42,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:47,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:32:47,376 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:32:47,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:32:47,378 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:32:47,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:32:47,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:32:47,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:32:47,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:32:47,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:48,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:32:51,216 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:32:53,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:58,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:32:58,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:33:02,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:02,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:02,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:33:02,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:33:03,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:08,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:33:08,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:13,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:17,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:17,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:17,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:33:17,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:33:18,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:33:19,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:21,216 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:33:25,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:26,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:33:30,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:32,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:32,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:33:32,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:33:35,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:40,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:45,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:46,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:33:47,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:47,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:33:47,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:33:47,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:33:51,217 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:33:51,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:56,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:33:57,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:01,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:03,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:34:03,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:34:03,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:34:03,512 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:34:06,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:07,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:12,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:16,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:18,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:18,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:34:18,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:34:21,219 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:34:24,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:26,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:29,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:33,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:34:33,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:34:34,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:34,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:39,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:44,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:45,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:48,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:34:48,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:34:50,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:34:51,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:34:54,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:34:55,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:00,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:03,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:35:03,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:35:04,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:35:06,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:12,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:12,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:35:17,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:18,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:35:18,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:35:18,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:35:18,515 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:35:21,220 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:35:22,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:22,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:35:27,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:32,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:35:33,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:33,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:35:33,512 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:35:38,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:42,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:35:43,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:48,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:49,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:35:49,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:35:49,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:35:49,031 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:35:51,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:35:52,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:35:54,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:35:59,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:00,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:04,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:36:04,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:36:04,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:36:04,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:36:04,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:09,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:09,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:36:09,654 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:36:09,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:36:09,657 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:36:09,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:36:09,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:10,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:14,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:19,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:36:19,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:36:19,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:36:19,028 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:36:20,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:21,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:21,222 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:36:26,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:31,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:31,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:34,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:36:34,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:36:37,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:41,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:42,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:47,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:49,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:36:49,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:36:49,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:36:49,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:51,222 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:36:53,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:36:58,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:36:58,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:03,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:04,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:37:04,027 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:37:08,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:37:09,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:14,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:18,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:37:19,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:37:19,026 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:37:20,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:21,223 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:37:25,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:28,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:37:31,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:34,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:37:34,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:37:34,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:37:36,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:36,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:37:41,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:46,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:37:47,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:49,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:37:49,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:37:49,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:37:51,223 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:37:53,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:37:56,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:37:58,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:03,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:04,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:38:04,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:38:06,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:38:09,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:14,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:38:15,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:19,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:38:19,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:38:20,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:21,225 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:38:24,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:38:25,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:30,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:34,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:38:34,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:38:34,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:38:34,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:38:34,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:38:36,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:41,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:43,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:38:47,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:50,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:38:50,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:38:50,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:38:50,802 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:38:51,228 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:38:52,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:38:53,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:38:57,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:03,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:03,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:05,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:39:05,799 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:39:08,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:13,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:14,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:19,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:19,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:39:19,981 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:39:19,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:39:19,984 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:39:20,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:39:20,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:39:20,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:39:21,228 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:39:21,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:22,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:25,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:30,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:31,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:35,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:35,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:39:35,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:39:35,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:39:40,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:41,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:45,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:50,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:39:50,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:39:50,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:39:51,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:51,229 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:39:56,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:39:58,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:40:02,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:06,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:40:06,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:40:06,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:40:06,481 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:40:07,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:08,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:40:12,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:18,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:18,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:40:21,229 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:40:21,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:40:21,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:40:21,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:40:21,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:40:23,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:28,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:40:28,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:33,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:36,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:40:36,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:40:36,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:40:36,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:40:39,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:44,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:46,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:40:49,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:51,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:40:51,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:40:51,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:40:51,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:40:55,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:40:56,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:41:00,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:05,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:06,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:41:06,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:41:06,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:41:07,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:41:10,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:15,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:21,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:41:21,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:21,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:41:21,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:41:21,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:41:21,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:41:26,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:31,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:41:32,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:36,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:41:36,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:41:36,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:41:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:43,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:48,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:51,231 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:41:51,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:41:51,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:41:51,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:41:53,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:41:58,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:04,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:05,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:42:06,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:42:06,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:42:06,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:42:09,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:15,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:42:15,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:20,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:21,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:42:21,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:42:21,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:42:21,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:42:23,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:42:26,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:31,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:35,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:42:36,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:42:36,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:42:36,480 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:42:36,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:41,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:42,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:42:47,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:51,235 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:42:51,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:42:51,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:42:51,481 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:42:52,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:42:52,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:42:57,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:01,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:43:01,447 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:43:01,448 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:43:01,450 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:43:01,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:43:02,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:43:03,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:06,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:43:06,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:43:06,480 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:43:08,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:12,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:43:13,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:18,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:20,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:43:21,234 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:43:21,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:43:21,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:43:21,480 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:43:24,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:29,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:31,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:43:34,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:36,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:43:36,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:43:36,483 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:43:40,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:41,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:43:45,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:51,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:51,234 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:43:51,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:43:51,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:43:51,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:43:51,642 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:43:56,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:43:59,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:44:02,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:07,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:44:07,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:44:07,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:44:07,286 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:44:08,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:09,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:44:13,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:19,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:19,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:44:21,235 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:44:22,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:44:22,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:44:22,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:44:24,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:27,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:44:30,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:35,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:37,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:44:37,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:44:37,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:44:37,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:44:41,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:47,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:47,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:44:51,235 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:44:52,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:52,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:44:52,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:44:52,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:44:52,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:44:57,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:44:57,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:02,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:07,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:45:07,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:45:07,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:45:07,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:08,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:13,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:15,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:18,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:21,238 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:45:22,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:45:22,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:45:22,796 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:45:23,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:24,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:29,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:34,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:34,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:37,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:45:37,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:45:37,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:45:39,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:44,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:45,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:49,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:45:51,239 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:45:52,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:45:52,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:45:52,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:45:55,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:45:55,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:00,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:03,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:46:05,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:07,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:46:07,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:46:07,794 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:46:11,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:12,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:46:12,497 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:46:12,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:46:12,500 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:46:13,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:46:13,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:46:16,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:21,239 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:46:21,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:22,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:46:22,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:46:22,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:46:23,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:46:26,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:32,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:33,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:46:37,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:37,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:46:37,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:46:37,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:46:42,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:43,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:46:47,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:51,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:46:51,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:46:52,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:46:52,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:46:52,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:46:53,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:46:58,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:01,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:47:03,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:07,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:47:07,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:47:07,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:47:08,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:09,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:47:13,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:19,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:19,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:47:21,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:47:22,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:47:22,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:47:22,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:47:25,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:29,844 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:47:30,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:35,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:37,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:47:37,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:47:37,796 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:47:39,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:47:41,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:46,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:47,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:47:51,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:47:51,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:52,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:47:52,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:47:52,798 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:47:56,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:47:58,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:02,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:07,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:07,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:07,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:48:07,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:48:07,796 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:48:12,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:18,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:21,241 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:48:24,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:24,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:48:24,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:48:24,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:48:24,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:48:27,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:29,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:34,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:35,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:39,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:48:39,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:48:39,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:45,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:45,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:50,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:48:51,242 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:48:54,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:48:54,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:48:55,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:48:55,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:00,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:03,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:49:05,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:09,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:49:09,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:49:09,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:49:11,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:13,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:49:16,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:21,244 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:49:21,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:21,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:49:21,960 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:49:21,960 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:49:21,963 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:49:22,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:49:23,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:49:24,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:49:24,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:49:24,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:49:27,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:33,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:33,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:49:38,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:39,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:49:39,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:49:39,506 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:49:41,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:49:43,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:48,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:51,244 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:49:51,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:49:54,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:49:54,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:49:54,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:49:54,505 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:49:59,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:01,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:50:04,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:09,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:50:09,504 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:50:10,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:11,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:50:15,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:20,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:50:21,245 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:50:21,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:24,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:50:24,505 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:50:26,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:30,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:50:31,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:36,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:39,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:50:39,505 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:50:40,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:50:42,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:48,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:50,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:50:51,246 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:50:54,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:54,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:50:54,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:50:54,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:50:54,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:50:59,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:50:59,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:04,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:07,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:09,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:51:09,505 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:51:09,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:14,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:17,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:20,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:21,246 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:51:24,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:51:24,507 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:51:25,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:27,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:30,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:35,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:35,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:39,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:51:39,506 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:51:41,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:45,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:47,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:51,248 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:51:53,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:51:54,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:51:54,506 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:51:55,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:51:58,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:03,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:52:04,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:09,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:09,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:52:09,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:52:09,507 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:52:09,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:52:13,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:52:14,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:19,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:21,248 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:52:23,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:52:24,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:52:24,506 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:52:25,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:30,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:31,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:52:31,861 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:52:31,862 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:52:31,864 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:52:31,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:52:33,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:52:35,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:40,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:52:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:52:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:52:40,357 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:52:41,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:44,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:52:46,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:51,249 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:52:52,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:52:52,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:52:55,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:52:55,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:52:55,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:52:55,356 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:52:57,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:02,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:53:02,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:07,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:10,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:53:10,357 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:53:12,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:53:13,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:18,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:20,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:53:21,249 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:53:24,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:25,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:25,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:25,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:53:25,357 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:53:29,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:34,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:39,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:40,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:53:40,358 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:53:42,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:53:45,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:51,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:53:51,250 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:53:52,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:53:55,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:55,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:53:55,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:53:55,364 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:53:56,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:01,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:54:01,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:06,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:10,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:54:10,364 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:54:11,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:54:12,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:17,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:21,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:54:21,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:54:23,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:25,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:25,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:25,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:54:25,364 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:54:28,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:31,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:54:34,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:39,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:54:40,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:40,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:40,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:54:40,364 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:54:40,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:45,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:49,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:54:50,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:51,251 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:54:55,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:55,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:54:55,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:54:55,366 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:54:56,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:54:59,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:55:02,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:07,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:10,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:55:10,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:55:10,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:55:10,365 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:55:12,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:18,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:55:18,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:21,251 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:55:24,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:25,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:55:25,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:55:25,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:55:28,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:55:29,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:34,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:38,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:55:39,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:40,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:55:40,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:55:40,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:55:44,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:48,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:55:50,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:55:51,252 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:55:55,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:55:55,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:55:55,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:55:55,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:00,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:56:00,744 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:56:00,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:56:00,746 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:56:01,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:56:01,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:01,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:02,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:06,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:10,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:56:10,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:56:10,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:56:10,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:56:12,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:14,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:17,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:21,252 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:56:22,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:22,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:25,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:56:25,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:56:27,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:32,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:33,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:38,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:40,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:56:40,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:56:42,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:43,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:48,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:51,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:56:51,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:54,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:56:55,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:56:55,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:56:59,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:56:59,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:04,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:09,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:57:10,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:10,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:57:10,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:57:15,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:19,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:57:21,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:57:21,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:26,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:57:26,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:57:26,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:57:26,147 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:57:26,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:29,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:57:32,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:37,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:40,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:57:41,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:57:41,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:57:41,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:57:41,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:57:42,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:48,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:48,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:57:51,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:57:53,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:57:56,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:57:56,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:57:56,148 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:57:58,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:57:58,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:03,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:08,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:58:09,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:11,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:58:11,145 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:58:14,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:18,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:58:19,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:21,254 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:58:25,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:26,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:58:26,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:58:28,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:58:30,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:36,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:36,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:58:41,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:41,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:58:41,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:58:46,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:58:46,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:51,255 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:58:52,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:58:56,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:58:56,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:58:56,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:58:57,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:02,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:06,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:07,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:11,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:59:11,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:59:11,824 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:59:12,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:12,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 05:59:12,855 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 05:59:12,856 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 05:59:12,859 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 05:59:13,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 05:59:13,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:14,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:17,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:21,256 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:59:23,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:24,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:26,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:59:26,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:59:26,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:59:29,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:34,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:34,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:39,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:41,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:59:41,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:59:41,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 05:59:44,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:45,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:50,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:51,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 05:59:52,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 05:59:55,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 05:59:56,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 05:59:56,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 05:59:56,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:00:00,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:02,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:00:05,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:11,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:11,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:00:11,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:00:11,824 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:00:12,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:00:16,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:20,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:00:21,257 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:00:21,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:26,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:27,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:00:27,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:00:27,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:00:27,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:00:30,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:00:32,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:37,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:40,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:00:42,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:42,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:00:42,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:00:42,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:00:42,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:00:48,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:50,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:00:51,258 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:00:53,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:00:57,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:00:57,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:00:57,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:00:59,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:00,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:04,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:08,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:09,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:12,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:01:12,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:01:12,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:01:12,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:01:15,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:18,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:20,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:21,258 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:01:26,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:27,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:01:27,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:01:27,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:01:27,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:01:28,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:32,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:37,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:38,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:42,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:43,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:01:43,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:01:43,999 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:01:46,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:48,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:51,259 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:01:53,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:01:57,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:01:58,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:01:58,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:01:58,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:01:59,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:04,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:07,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:09,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:13,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:02:13,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:02:13,998 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:02:13,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:02:15,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:16,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:20,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:21,260 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:02:23,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:02:23,514 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:02:23,514 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:02:23,516 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:02:24,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:02:24,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:25,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:26,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:28,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:02:28,997 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:02:31,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:34,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:36,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:41,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:43,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:02:43,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:02:43,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:02:43,998 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:02:44,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:51,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:02:52,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:54,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:02:57,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:02:58,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:02:58,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:02:58,998 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:03:03,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:04,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:03:08,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:12,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:03:13,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:14,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:14,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:14,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:03:14,042 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:03:19,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:21,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:03:22,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:03:25,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:29,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:29,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:03:29,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:03:30,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:32,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:03:35,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:41,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:42,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:03:44,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:44,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:44,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:03:44,398 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:03:46,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:51,262 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:03:52,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:52,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:03:57,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:03:59,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:59,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:03:59,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:03:59,398 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:04:00,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:02,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:07,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:10,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:13,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:14,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:04:14,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:04:14,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:04:18,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:21,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:21,263 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:04:24,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:29,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:04:29,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:04:29,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:04:29,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:31,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:34,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:40,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:41,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:44,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:04:44,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:04:44,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:04:45,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:50,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:51,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:51,263 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:04:56,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:04:59,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:04:59,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:04:59,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:04:59,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:05:01,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:06,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:09,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:05:11,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:14,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:05:14,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:05:14,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:05:17,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:18,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:05:21,264 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:05:22,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:26,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:05:27,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:29,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:05:29,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:05:29,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:05:33,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:05:35,485 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:05:35,486 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:05:35,488 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:05:35,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:05:36,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:05:38,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:43,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:44,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:05:44,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:05:44,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:05:44,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:05:46,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:05:49,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:51,264 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:05:54,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:56,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:05:59,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:05:59,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:05:59,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:05:59,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:06:04,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:04,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:06:09,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:14,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:06:14,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:06:14,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:06:14,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:06:15,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:20,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:21,266 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:06:24,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:06:25,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:30,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:31,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:06:31,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:06:31,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:06:34,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:06:36,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:41,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:42,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:06:46,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:06:46,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:06:46,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:06:46,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:51,266 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:06:51,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:06:53,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:06:56,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:01,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:07:01,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:07:01,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:07:02,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:03,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:07:07,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:13,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:07:13,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:16,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:07:16,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:07:16,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:07:18,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:21,267 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:07:23,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:07:24,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:29,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:31,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:07:31,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:07:31,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:07:31,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:07:34,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:40,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:41,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:07:45,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:47,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:07:47,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:07:47,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:07:47,598 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:07:50,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:07:51,267 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:07:51,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:07:56,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:01,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:01,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:02,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:08:02,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:08:06,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:08,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:12,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:17,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:08:17,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:08:17,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:18,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:21,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:08:23,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:26,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:29,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:32,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:08:32,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:08:32,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:08:32,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:08:34,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:36,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:40,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:40,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:45,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:46,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:08:46,349 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:08:46,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:08:46,352 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:08:46,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:08:47,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:08:47,596 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:08:48,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:08:50,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:51,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:08:55,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:08:56,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:00,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:04,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:09:04,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:09:04,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:09:06,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:07,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:11,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:16,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:17,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:19,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:09:19,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:09:19,066 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:09:21,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:09:22,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:25,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:28,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:33,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:34,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:09:34,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:09:34,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:09:34,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:09:35,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:38,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:43,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:47,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:48,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:49,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:09:49,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:09:51,271 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:09:54,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:09:55,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:09:59,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:04,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:10:04,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:10:05,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:09,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:10:11,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:16,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:18,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:10:19,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:10:19,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:10:21,282 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:10:22,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:27,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:28,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:10:32,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:34,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:10:34,079 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:10:37,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:38,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:10:42,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:48,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:48,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:10:49,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:10:49,077 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:10:51,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:10:54,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:10:57,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:00,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:04,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:11:04,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:11:04,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:11:04,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:11:05,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:07,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:10,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:16,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:17,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:19,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:11:19,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:11:19,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:11:21,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:11:22,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:27,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:28,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:33,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:34,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:11:34,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:11:34,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:11:34,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:11:37,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:38,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:43,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:45,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:49,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:49,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:11:49,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:11:49,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:11:51,274 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:11:54,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:11:55,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:11:59,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:03,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:12:03,137 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:12:03,138 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:12:03,142 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:12:03,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:12:04,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:04,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:12:04,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:12:04,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:12:05,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:05,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:10,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:13,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:16,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:12:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:12:20,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:12:20,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:12:21,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:12:21,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:23,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:26,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:32,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:33,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:35,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:12:35,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:12:35,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:12:38,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:43,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:48,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:50,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:12:50,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:12:50,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:12:50,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:12:51,274 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:12:53,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:12:54,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:12:59,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:01,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:13:05,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:05,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:13:05,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:13:05,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:13:10,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:10,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:13:15,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:20,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:13:20,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:13:20,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:13:20,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:13:21,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:21,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:13:26,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:31,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:13:32,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:37,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:13:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:13:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:13:37,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:13:39,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:13:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:47,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:49,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:13:51,276 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:13:52,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:13:52,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:13:52,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:58,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:13:59,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:03,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:07,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:14:07,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:14:08,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:09,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:13,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:17,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:19,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:21,276 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:14:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:14:22,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:14:22,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:14:24,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:27,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:30,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:35,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:14:37,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:14:37,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:40,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:45,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:47,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:50,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:14:51,277 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:14:53,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:14:53,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:14:53,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:14:53,158 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:14:55,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:14:56,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:01,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:05,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:06,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:08,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:15:08,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:15:08,158 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:15:12,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:13,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:15:13,411 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:15:13,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:15:13,414 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:15:13,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:15:14,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:15,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:18,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:21,279 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:15:23,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:15:23,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:15:23,158 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:15:23,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:24,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:29,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:33,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:34,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:38,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:15:38,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:15:38,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:15:40,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:43,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:45,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:50,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:15:51,280 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:15:53,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:15:53,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:15:53,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:15:53,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:15:54,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:15:56,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:01,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:04,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:16:06,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:08,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:08,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:08,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:16:08,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:16:11,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:13,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:16:16,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:21,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:16:21,280 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:16:22,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:23,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:23,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:16:23,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:16:27,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:31,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:16:32,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:37,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:38,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:38,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:38,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:16:38,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:16:41,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:16:43,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:48,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:49,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:16:51,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:16:53,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:53,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:16:53,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:16:53,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:16:53,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:58,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:16:59,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:03,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:08,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:17:08,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:17:08,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:17:09,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:09,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:14,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:19,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:20,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:21,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:17:23,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:17:23,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:17:23,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:17:23,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:17:25,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:27,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:30,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:35,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:37,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:38,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:17:38,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:17:38,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:17:41,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:47,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:47,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:51,282 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:17:53,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:17:54,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:17:54,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:17:54,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:17:57,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:17:58,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:03,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:07,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:09,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:09,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:18:09,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:18:09,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:18:14,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:16,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:19,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:21,283 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:18:23,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:18:23,815 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:18:23,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:18:23,817 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:18:24,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:18:24,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:24,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:18:24,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:18:24,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:18:25,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:26,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:30,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:35,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:36,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:39,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:18:39,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:18:39,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:18:39,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:18:41,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:44,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:47,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:51,284 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:18:52,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:18:54,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:18:54,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:18:54,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:18:54,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:18:54,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:18:58,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:03,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:19:03,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:08,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:10,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:19:10,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:19:10,035 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:19:13,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:19:14,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:19,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:21,285 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:19:23,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:19:24,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:25,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:19:25,034 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:19:30,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:33,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:19:35,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:40,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:19:40,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:19:40,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:19:40,389 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:19:41,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:43,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:19:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:51,285 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:19:51,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:19:52,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:19:55,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:19:55,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:19:55,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:19:57,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:01,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:20:03,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:08,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:10,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:20:10,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:20:10,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:20:11,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:20:14,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:19,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:20:20,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:21,285 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:20:25,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:25,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:20:25,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:20:25,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:20:30,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:20:30,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:35,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:40,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:20:40,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:20:40,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:20:40,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:20:40,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:45,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:50,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:20:51,286 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:20:51,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:56,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:20:56,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:20:56,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:20:56,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:20:56,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:21:00,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:01,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:06,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:10,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:11,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:21:11,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:21:11,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:21:11,838 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:21:12,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:17,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:18,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:21,286 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:21:23,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:26,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:21:26,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:21:26,847 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:21:27,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:29,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:34,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:36,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:21:36,369 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:21:36,370 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:21:36,372 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:21:36,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:21:37,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:39,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:41,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:21:41,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:21:41,838 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:21:45,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:47,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:50,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:51,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:21:55,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:21:55,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:21:56,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:21:56,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:21:56,839 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:22:01,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:05,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:22:06,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:11,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:13,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:22:13,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:22:13,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:22:13,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:22:15,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:22:17,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:21,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:22:23,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:25,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:22:28,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:22:28,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:22:28,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:33,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:36,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:22:38,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:43,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:22:43,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:22:43,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:22:43,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:22:43,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:46,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:22:48,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:51,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:22:54,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:22:54,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:22:58,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:22:58,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:22:59,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:04,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:23:04,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:09,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:13,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:23:13,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:23:13,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:23:14,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:23:15,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:20,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:21,291 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:23:24,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:23:25,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:28,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:23:28,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:23:28,262 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:23:28,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:23:31,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:32,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:23:36,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:41,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:42,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:23:43,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:23:43,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:23:47,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:51,291 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:23:52,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:23:53,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:58,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:23:58,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:23:58,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:23:58,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:23:58,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:24:02,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:24:04,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:09,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:12,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:24:13,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:24:13,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:24:13,991 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:24:14,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:19,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:21,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:24:24,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:26,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:24:28,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:24:28,991 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:24:30,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:33,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:24:35,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:40,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:43,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:24:43,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:24:43,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:24:46,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:51,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:51,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:24:52,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:24:52,380 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:24:52,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:24:52,384 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:24:52,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:24:53,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:24:56,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:24:58,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:24:58,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:25:02,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:04,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:25:07,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:12,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:13,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:25:13,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:25:14,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:25:18,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:21,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:25:22,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:25:23,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:28,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:29,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:25:29,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:25:29,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:25:32,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:25:33,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:38,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:42,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:25:44,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:25:44,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:25:44,727 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:25:44,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:49,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:51,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:25:52,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:25:55,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:25:59,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:25:59,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:25:59,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:25:59,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:26:00,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:02,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:26:05,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:10,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:26:11,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:14,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:26:14,727 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:26:16,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:20,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:26:21,306 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:26:22,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:27,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:30,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:26:30,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:26:30,415 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:26:32,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:26:32,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:37,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:42,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:26:43,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:45,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:26:45,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:26:45,415 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:26:48,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:51,296 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:26:52,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:26:54,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:26:59,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:00,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:27:00,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:27:00,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:27:00,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:27:04,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:27:04,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:09,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:14,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:27:15,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:27:15,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:27:15,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:27:16,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:21,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:21,296 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:27:23,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:27:26,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:30,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:27:30,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:27:30,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:27:32,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:34,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:27:37,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:42,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:27:42,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:45,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:27:45,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:27:48,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:51,297 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:27:52,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:27:53,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:27:58,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:00,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:28:00,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:28:04,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:09,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:12,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:28:14,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:15,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:28:15,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:28:20,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:20,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:28:20,891 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:28:20,891 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:28:20,894 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:28:21,297 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:28:21,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:28:22,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:28:25,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:26,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:28:30,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:31,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:28:31,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:28:31,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:28:31,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:28:34,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:28:36,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:41,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:44,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:28:46,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:28:46,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:28:46,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:28:47,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:51,299 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:28:52,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:28:54,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:28:57,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:02,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:02,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:02,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:29:02,443 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:29:02,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:04,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:29:07,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:12,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:16,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:29:17,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:17,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:17,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:29:17,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:29:17,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:21,302 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:29:23,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:26,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:29:28,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:32,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:32,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:32,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:29:32,944 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:29:34,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:36,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:29:39,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:45,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:46,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:29:47,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:29:47,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:29:47,943 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:29:50,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:51,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:29:55,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:29:57,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:00,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:03,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:30:03,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:30:03,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:30:03,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:30:05,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:07,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:10,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:15,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:17,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:18,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:30:18,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:30:18,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:30:18,158 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:30:21,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:30:21,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:26,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:27,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:32,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:33,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:30:33,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:30:33,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:30:36,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:37,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:42,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:46,371 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:47,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:48,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:30:48,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:30:48,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:30:51,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:30:52,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:30:56,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:30:57,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:02,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:04,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:31:04,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:31:04,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:31:06,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:31:08,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:13,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:16,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:31:18,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:19,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:31:19,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:31:19,072 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:31:21,304 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:31:24,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:26,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:31:29,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:34,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:31:34,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:31:34,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:31:34,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:31:35,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:36,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:31:40,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:44,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:31:44,538 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:31:44,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:31:44,542 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:31:44,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:31:45,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:45,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:31:46,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:31:49,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:31:49,072 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:31:51,306 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:31:51,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:56,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:31:56,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:32:01,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:04,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:32:04,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:32:06,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:32:07,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:12,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:17,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:32:17,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:19,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:32:19,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:32:21,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:32:23,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:27,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:32:28,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:33,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:34,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:32:34,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:32:37,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:32:38,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:43,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:47,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:32:48,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:49,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:32:49,073 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:32:51,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:32:54,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:32:57,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:00,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:04,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:33:04,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:33:04,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:33:04,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:33:05,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:07,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:10,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:16,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:16,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:19,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:33:19,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:33:21,308 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:33:22,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:26,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:27,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:32,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:34,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:33:34,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:33:34,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:33:36,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:38,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:43,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:46,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:48,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:49,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:33:49,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:33:49,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:33:51,309 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:33:54,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:33:54,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:33:59,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:04,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:34:04,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:34:04,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:34:04,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:34:05,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:10,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:14,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:34:15,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:19,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:34:19,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:34:19,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:34:21,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:21,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:34:24,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:34:26,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:31,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:34,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:34:34,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:34:34,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:34:35,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:34:37,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:42,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:45,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:34:47,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:49,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:34:49,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:34:49,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:34:51,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:34:53,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:34:55,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:34:58,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:02,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:35:02,731 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:35:02,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:35:02,733 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:35:03,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:35:03,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:04,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:35:04,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:35:04,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:35:04,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:35:05,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:35:09,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:14,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:19,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:35:19,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:35:19,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:35:19,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:21,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:35:24,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:29,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:34,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:35:34,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:35:34,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:35:35,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:40,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:45,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:49,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:35:49,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:35:49,083 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:35:51,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:35:51,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:35:56,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:01,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:04,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:36:04,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:36:04,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:36:06,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:11,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:16,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:19,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:36:19,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:36:19,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:36:21,313 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:36:22,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:27,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:32,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:34,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:36:34,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:36:34,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:36:38,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:43,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:48,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:49,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:36:49,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:36:49,083 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:36:51,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:36:53,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:36:58,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:03,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:04,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:37:04,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:37:04,083 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:37:09,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:14,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:19,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:19,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:37:19,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:37:19,083 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:37:21,317 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:37:24,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:29,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:34,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:37:34,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:37:34,083 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:37:35,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:40,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:45,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:49,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:37:49,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:37:49,083 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:37:50,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:37:51,317 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:37:55,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:00,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:38:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:38:04,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:38:06,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:11,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:16,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:19,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:38:19,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:38:19,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:38:21,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:38:21,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:26,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:31,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:34,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:38:34,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:38:34,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:38:36,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:41,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:46,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:49,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:38:49,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:38:49,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:38:51,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:38:52,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:38:57,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:02,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:39:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:39:04,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:39:08,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:13,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:18,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:19,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:39:19,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:39:19,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:39:21,319 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:39:24,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:29,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:34,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:39:34,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:39:34,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:39:35,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:40,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:45,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:49,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:39:49,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:39:49,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:39:50,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:39:51,328 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:39:55,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:00,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:04,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:40:04,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:40:04,085 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:40:05,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:10,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:15,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:19,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:40:19,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:40:19,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:40:19,085 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:40:21,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:21,328 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:40:26,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:31,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:34,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:40:34,085 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:40:37,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:42,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:47,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:49,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:40:49,085 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:40:51,329 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:40:52,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:40:57,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:02,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:04,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:41:04,085 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:41:08,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:13,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:18,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:19,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:41:19,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:41:19,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:41:19,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:41:21,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:41:24,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:29,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:34,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:41:34,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:41:35,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:40,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:45,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:49,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:41:49,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:41:50,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:41:51,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:41:55,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:00,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:04,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:42:04,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:42:06,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:11,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:16,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:19,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:42:19,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:42:21,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:21,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:42:26,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:31,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:34,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:42:34,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:42:36,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:41,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:46,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:49,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:42:49,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:42:51,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:42:52,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:42:57,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:02,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:04,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:43:04,086 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:43:08,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:13,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:18,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:19,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:43:19,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:43:21,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:43:23,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:28,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:33,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:34,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:43:34,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:43:39,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:44,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:49,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:43:49,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:43:49,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:51,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:43:54,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:43:59,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:04,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:44:04,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:44:05,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:10,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:15,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:19,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:44:19,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:44:20,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:21,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:44:25,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:30,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:34,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:44:34,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:44:35,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:40,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:45,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:49,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:44:49,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:44:50,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:44:51,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:44:56,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:01,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:04,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:45:04,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:45:07,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:12,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:17,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:19,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:45:19,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:45:21,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:45:23,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:28,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:33,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:33,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:45:33,771 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:45:33,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:45:33,775 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:45:34,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:45:34,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:45:34,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:45:35,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:45:36,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:45:39,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:44,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:49,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:45:49,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:45:49,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:51,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:45:54,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:45:59,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:00,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:46:04,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:46:04,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:46:04,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:46:05,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:10,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:15,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:19,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:46:19,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:46:19,256 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:46:21,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:46:21,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:26,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:31,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:34,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:46:34,216 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:46:34,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:46:36,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:39,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:46:40,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:46:41,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:46,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:48,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:46:51,335 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:46:52,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:46:54,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:46:54,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:46:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:02,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:07,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:09,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:47:09,348 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:47:09,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:47:09,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:47:11,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:47:12,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:17,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:21,335 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:47:23,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:24,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:47:24,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:47:24,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:47:28,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:33,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:38,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:39,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:47:39,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:47:39,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:47:44,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:49,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:47:51,335 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:47:54,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:47:54,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:47:54,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:47:55,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:00,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:05,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:09,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:48:09,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:48:09,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:48:10,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:12,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:48:15,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:20,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:21,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:48:24,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:48:24,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:48:24,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:48:24,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:48:26,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:31,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:34,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:48:37,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:39,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:48:39,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:48:39,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:48:42,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:45,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:48:48,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:51,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:48:53,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:48:54,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:48:54,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:48:54,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:48:55,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:48:58,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:04,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:05,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:49:09,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:49:09,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:49:09,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:49:10,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:15,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:49:15,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:49:16,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:21,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:49:22,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:24,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:49:24,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:49:24,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:49:26,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:49:27,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:32,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:34,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:49:38,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:39,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:49:39,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:49:39,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:49:44,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:49:44,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:44,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:49:49,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:51,340 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:49:54,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:49:54,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:49:54,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:49:54,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:49:54,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:49:59,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:04,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:50:05,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:09,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:09,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:50:09,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:50:10,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:14,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:50:14,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:16,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:21,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:21,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:50:24,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:50:24,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:24,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:50:24,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:50:26,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:29,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:31,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:34,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:50:36,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:39,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:39,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:50:39,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:50:42,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:44,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:50:44,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:47,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:51,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:50:53,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:54,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:50:54,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:50:54,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:50:54,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:50:58,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:50:59,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:04,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:04,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:51:09,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:09,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:09,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:51:09,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:51:14,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:14,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:51:15,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:19,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:21,344 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:51:24,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:24,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:51:24,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:51:24,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:51:24,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:29,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:30,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:33,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:51:33,799 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:51:33,799 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:51:33,804 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:51:33,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:51:35,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:35,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:51:39,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:39,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:51:39,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:51:41,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:45,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:51:46,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:51,345 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:51:52,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:54,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:51:54,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:51:54,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:51:56,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:51:57,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:51:59,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:02,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:06,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:52:08,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:09,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:09,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:52:09,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:52:13,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:15,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:16,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:52:19,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:21,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:52:24,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:24,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:24,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:52:24,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:52:29,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:30,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:32,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:52:35,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:39,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:39,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:52:39,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:52:40,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:42,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:52:45,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:45,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:51,346 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:52:51,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:52:52,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:52:54,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:52:54,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:52:54,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:52:56,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:00,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:02,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:02,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:53:07,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:09,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:09,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:53:09,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:53:12,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:53:13,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:15,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:18,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:21,355 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:53:22,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:53:23,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:24,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:24,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:53:24,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:53:28,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:30,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:32,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:53:34,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:39,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:39,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:39,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:53:39,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:53:42,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:53:45,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:45,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:50,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:53:51,356 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:53:52,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:53:54,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:53:54,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:53:54,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:53:55,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:00,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:00,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:02,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:54:05,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:09,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:54:09,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:09,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:54:11,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:12,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:54:15,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:16,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:21,356 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:54:22,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:54:22,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:24,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:54:24,918 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:54:28,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:30,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:33,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:34,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:54:38,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:40,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:54:40,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:54:44,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:54:44,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:45,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:49,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:51,357 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:54:52,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:54:55,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:54:55,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:54:55,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:54:55,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:55:00,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:00,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:01,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:55:01,443 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:55:01,444 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:55:01,447 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:55:02,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:55:02,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:03,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:06,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:10,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:10,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:55:10,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:55:12,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:12,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:15,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:17,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:21,358 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:55:22,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:22,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:25,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:55:25,590 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:55:27,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:31,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:33,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:34,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:38,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:40,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:40,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:55:40,797 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:55:43,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:44,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:46,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:48,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:51,371 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:55:54,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:55:54,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:55:55,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:55:55,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:55:55,798 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:55:59,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:01,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:04,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:56:05,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:10,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:10,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:10,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:56:10,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:56:14,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:56:15,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:16,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:20,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:21,359 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:56:24,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:56:25,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:25,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:56:25,798 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:56:25,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:30,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:31,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:35,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:56:36,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:40,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:40,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:56:40,799 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:56:42,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:45,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:56:46,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:47,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:51,360 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:56:52,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:56:55,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:56:55,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:56:55,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:56:55,797 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:56:58,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:01,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:03,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:04,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:57:08,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:10,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:10,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:57:10,801 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:57:13,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:14,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:57:16,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:21,361 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:57:24,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:24,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:57:25,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:57:25,798 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:57:29,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:31,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:34,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:57:35,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:40,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:41,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:41,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:57:41,392 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:57:44,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:57:46,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:47,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:51,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:57:52,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:57:54,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:57:56,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:57:56,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:57:56,392 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:57:57,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:02,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:58:03,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:04,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:08,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:11,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:58:11,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:58:11,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:58:14,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:15,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:17,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:58:19,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:21,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:58:22,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 06:58:22,284 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 06:58:22,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 06:58:22,288 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 06:58:23,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 06:58:24,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:25,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:25,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:26,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:58:26,392 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:58:30,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:32,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:58:35,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:36,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:41,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:41,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:58:41,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:58:41,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:58:45,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:47,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:47,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:58:51,365 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:58:52,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:58:55,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:58:56,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:58:56,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:58:57,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:02,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:59:03,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:04,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:59:08,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:11,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:59:11,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:59:11,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:59:12,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:59:13,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:17,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:59:18,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:21,365 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:59:22,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:59:24,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:26,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:59:26,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:59:29,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:32,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:59:32,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:59:35,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:40,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:41,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:59:41,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:59:41,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 06:59:42,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:59:45,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:47,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 06:59:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:51,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 06:59:52,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 06:59:56,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 06:59:56,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 06:59:56,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:00:01,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:02,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:00:02,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:00:06,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:11,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:00:11,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:00:11,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:12,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:00:16,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:17,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:00:21,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:00:22,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:22,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:00:26,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:00:26,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:00:27,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:31,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:00:32,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:00:32,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:37,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:41,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:00:41,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:00:43,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:00:43,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:47,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:00:48,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:51,367 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:00:53,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:00:54,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:00:56,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:00:56,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:00:59,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:01,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:02,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:01:05,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:10,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:11,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:11,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:01:11,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:01:16,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:18,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:01:21,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:21,367 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:01:22,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:26,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:01:26,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:01:27,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:31,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:32,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:01:32,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:37,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:39,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:01:39,592 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:01:39,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:01:39,595 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:01:40,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:01:41,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:01:41,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:01:41,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:42,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:43,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:48,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:01:48,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:51,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:01:51,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:01:54,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:01:56,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:01:56,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:02:00,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:00,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:02:02,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:02:05,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:10,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:02:10,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:11,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:02:11,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:02:16,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:17,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:02:20,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:02:21,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:02:22,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:27,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:02:27,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:02:27,333 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:02:27,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:30,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:02:32,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:02:33,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:38,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:40,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:02:42,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:02:42,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:02:42,332 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:02:43,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:47,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:02:49,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:50,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:02:51,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:02:54,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:02:57,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:02:57,811 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:03:00,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:01,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:03:03,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:03:05,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:10,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:11,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:03:12,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:03:12,811 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:03:16,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:18,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:03:19,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:03:21,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:21,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:03:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:27,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:03:27,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:03:29,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:03:32,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:33,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:03:37,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:39,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:03:42,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:42,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:03:42,811 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:03:47,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:48,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:03:49,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:03:51,370 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:03:53,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:57,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:03:57,811 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:03:58,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:03:59,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:03,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:04:03,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:09,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:09,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:12,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:04:12,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:04:15,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:18,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:04:19,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:20,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:21,370 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:04:26,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:27,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:04:27,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:04:29,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:32,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:33,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:04:37,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:38,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:42,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:42,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:04:42,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:04:47,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:48,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:04:48,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:51,370 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:04:53,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:04:57,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:04:57,436 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:04:57,436 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:04:57,438 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:04:57,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:04:57,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:04:58,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:04:58,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:59,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:04:59,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:03,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:05:04,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:09,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:09,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:05:12,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:05:12,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:05:15,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:18,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:05:19,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:05:20,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:21,371 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:05:26,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:27,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:05:27,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:05:29,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:05:31,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:33,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:05:36,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:39,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:05:42,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:42,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:05:42,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:05:47,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:48,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:05:49,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:05:51,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:05:53,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:05:57,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:05:57,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:05:57,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:05:59,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:04,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:05,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:06:07,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:06:09,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:12,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:06:12,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:06:15,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:17,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:06:18,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:06:20,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:21,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:06:25,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:27,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:06:27,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:06:27,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:06:31,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:35,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:06:36,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:37,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:06:41,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:42,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:06:42,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:06:46,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:48,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:06:49,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:06:51,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:06:52,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:57,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:06:57,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:06:58,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:06:59,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:07:03,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:03,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:07:08,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:12,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:07:12,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:07:13,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:07:14,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:18,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:07:19,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:21,374 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:07:23,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:07:24,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:27,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:07:27,814 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:07:29,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:33,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:07:33,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:07:35,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:40,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:42,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:07:42,814 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:07:43,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:07:45,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:50,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:07:50,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:51,379 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:07:53,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:07:56,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:07:57,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:07:57,816 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:08:01,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:01,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:03,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:08:06,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:12,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:12,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:08:12,814 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:08:13,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:18,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:19,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:08:20,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:08:20,688 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:08:20,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:08:20,690 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:08:20,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:08:21,375 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:08:21,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:22,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:23,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:28,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:29,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:08:29,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:08:29,497 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:08:31,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:33,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:34,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:08:38,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:41,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:44,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:08:44,494 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:08:44,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:49,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:50,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:08:51,375 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:08:51,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:08:55,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:08:59,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:08:59,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:08:59,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:09:01,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:02,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:09:05,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:09:06,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:11,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:12,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:09:14,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:09:14,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:09:16,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:20,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:09:21,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:09:21,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:09:22,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:27,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:29,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:09:29,497 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:09:31,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:09:32,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:35,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:09:37,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:41,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:09:43,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:44,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:09:44,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:09:48,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:50,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:09:51,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:09:51,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:09:54,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:59,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:09:59,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:09:59,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:09:59,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:10:01,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:10:04,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:05,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:10:09,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:11,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:10:14,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:10:14,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:10:15,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:20,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:10:20,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:21,377 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:10:21,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:10:26,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:29,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:10:29,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:10:29,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:10:31,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:10:32,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:35,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:10:37,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:41,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:10:42,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:44,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:10:44,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:10:48,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:50,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:10:51,377 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:10:51,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:10:54,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:59,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:10:59,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:10:59,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:11:01,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:05,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:11:10,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:12,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:14,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:11:14,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:11:16,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:20,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:11:21,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:11:22,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:22,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:27,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:29,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:11:29,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:11:32,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:32,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:35,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:11:37,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:38,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:11:38,907 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:11:38,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:11:38,910 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:11:39,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:11:40,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:41,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:42,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:44,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:11:44,495 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:11:48,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:50,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:11:50,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:11:51,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:11:54,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:11:59,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:11:59,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:11:59,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:00,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:12:04,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:05,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:12:09,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:12:09,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:14,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:12:14,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:12:15,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:19,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:12:20,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:12:21,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:21,379 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:12:26,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:29,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:12:29,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:12:29,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:12:31,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:35,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:12:36,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:39,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:12:42,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:44,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:12:44,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:12:47,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:49,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:12:50,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:12:51,379 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:12:53,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:58,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:12:59,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:12:59,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:12:59,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:03,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:06,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:13:09,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:09,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:14,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:13:14,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:13:14,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:13:14,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:17,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:20,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:13:20,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:21,379 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:13:25,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:28,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:29,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:13:29,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:13:30,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:35,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:13:35,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:38,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:41,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:44,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:13:44,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:13:44,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:13:47,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:48,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:50,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:13:51,380 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:13:52,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:58,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:13:58,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:13:59,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:13:59,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:14:04,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:05,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:14:08,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:09,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:14,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:14,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:14:14,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:14:14,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:14:18,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:20,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:20,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:14:21,381 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:14:26,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:26,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:29,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:14:29,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:14:32,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:35,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:14:36,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:37,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:42,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:44,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:14:44,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:14:45,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:48,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:50,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:14:51,383 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:14:53,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:54,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:14:54,175 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:14:54,176 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:14:54,178 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:14:54,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:14:54,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:55,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:14:59,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:14:59,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:14:59,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:14:59,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:15:04,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:05,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:15:05,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:15:09,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:14,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:14,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:15:14,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:15:15,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:15:20,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:20,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:15:21,384 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:15:25,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:25,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:15:29,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:15:29,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:15:31,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:35,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:15:35,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:15:36,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:41,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:44,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:15:44,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:15:46,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:15:47,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:50,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:15:51,385 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:15:52,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:56,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:15:57,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:15:59,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:15:59,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:16:03,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:05,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:16:06,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:16:08,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:14,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:16:14,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:14,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:16:14,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:16:20,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:21,386 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:16:21,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:16:24,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:16:25,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:29,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:16:29,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:16:31,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:34,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:16:36,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:16:37,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:42,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:44,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:16:44,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:16:44,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:16:47,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:51,387 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:16:51,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:16:52,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:16:54,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:16:57,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:01,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:01,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:17:01,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:17:02,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:04,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:17:06,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:07,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:13,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:14,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:17:16,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:16,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:17:16,508 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:17:18,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:21,389 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:17:21,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:24,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:24,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:17:29,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:31,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:31,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:17:31,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:17:34,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:17:34,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:36,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:39,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:41,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:17:45,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:46,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:46,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:17:46,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:17:50,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:17:51,389 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:17:51,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:17:51,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:17:56,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:01,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:01,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:18:01,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:18:01,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:01,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:06,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:06,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:10,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:18:10,333 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:18:10,334 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:18:10,337 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:18:11,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:18:11,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:12,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:12,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:16,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:16,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:18:16,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:18:17,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:21,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:18:21,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:22,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:28,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:31,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:31,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:18:31,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:18:32,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:33,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:36,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:38,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:42,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:43,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:46,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:46,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:18:46,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:18:49,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:18:51,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:18:51,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:18:52,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:18:55,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:00,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:01,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:01,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:19:01,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:19:02,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:19:06,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:08,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:10,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:19:12,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:17,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:17,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:17,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:19:17,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:19:20,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:19:21,391 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:19:23,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:23,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:28,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:30,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:19:32,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:32,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:19:32,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:19:34,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:38,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:40,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:40,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:19:45,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:47,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:47,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:19:47,928 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:19:50,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:50,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:19:51,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:19:53,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:19:55,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:19:58,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:00,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:02,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:02,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:20:02,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:20:06,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:08,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:08,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:11,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:16,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:17,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:17,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:20:17,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:20:18,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:21,403 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:20:22,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:23,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:27,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:28,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:32,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:32,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:32,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:20:32,927 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:20:38,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:38,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:38,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:43,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:47,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:47,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:20:47,928 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:20:48,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:48,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:51,393 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:20:53,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:20:54,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:20:56,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:20:58,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:03,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:04,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:04,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:21:04,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:21:06,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:21:09,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:09,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:14,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:16,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:21:19,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:19,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:21:19,119 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:21:19,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:21,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:21:24,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:21:24,163 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:21:24,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:21:24,166 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:21:24,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:24,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:21:25,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:25,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:21:26,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:21:30,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:34,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:34,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:21:34,119 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:21:35,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:39,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:40,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:45,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:49,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:49,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:21:49,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:21:51,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:21:51,394 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:21:54,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:21:56,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:02,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:02,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:22:04,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:04,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:22:04,119 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:22:08,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:09,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:12,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:22:13,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:18,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:19,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:19,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:22:19,123 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:22:21,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:22:23,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:22:24,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:24,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:29,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:33,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:22:34,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:34,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:22:34,120 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:22:35,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:39,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:41,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:43,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:22:46,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:49,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:49,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:22:49,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:22:51,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:22:51,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:22:53,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:22:54,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:22:56,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:00,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:23:01,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:04,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:04,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:23:04,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:23:07,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:09,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:10,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:23:12,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:17,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:19,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:19,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:23:19,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:23:20,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:23:21,396 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:23:23,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:24,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:28,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:30,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:23:33,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:34,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:23:34,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:23:39,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:39,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:40,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:23:44,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:49,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:49,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:23:49,159 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:23:49,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:23:50,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:23:51,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:23:54,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:23:55,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:00,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:01,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:04,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:04,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:24:04,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:24:06,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:09,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:10,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:11,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:16,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:19,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:19,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:24:19,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:24:20,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:21,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:24:22,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:24,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:27,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:28,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:33,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:24:34,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:24:38,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:38,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:39,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:43,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:49,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:49,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:49,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:24:49,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:24:49,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:51,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:24:54,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:24:55,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:24:59,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:24:59,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:04,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:05,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:05,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:25:05,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:25:06,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:25:06,412 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:25:06,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:25:06,416 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:25:07,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:25:07,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:09,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:10,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:10,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:15,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:17,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:20,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:20,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:25:20,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:25:20,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:21,399 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:25:25,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:25,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:27,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:30,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:35,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:35,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:25:35,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:25:36,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:37,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:40,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:41,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:46,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:47,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:50,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:50,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:25:50,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:25:51,399 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:25:52,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:25:55,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:25:56,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:25:57,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:02,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:05,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:05,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:26:05,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:26:06,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:26:08,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:10,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:13,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:16,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:26:18,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:20,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:20,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:26:20,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:26:21,401 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:26:24,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:24,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:26:25,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:29,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:34,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:26:34,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:35,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:35,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:26:35,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:26:40,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:40,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:44,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:26:46,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:50,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:50,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:26:50,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:26:51,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:26:51,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:26:54,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:26:55,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:26:56,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:01,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:05,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:27:05,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:05,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:27:05,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:27:07,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:11,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:12,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:15,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:27:17,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:21,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:21,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:27:21,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:27:21,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:27:23,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:27:23,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:26,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:28,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:33,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:35,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:27:36,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:36,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:27:36,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:27:39,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:41,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:44,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:45,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:27:49,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:51,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:27:51,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:27:51,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:27:51,556 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:27:55,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:27:55,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:27:56,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:00,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:03,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:06,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:06,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:28:06,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:28:06,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:11,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:11,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:13,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:17,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:21,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:21,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:28:21,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:28:21,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:28:21,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:28:21,777 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:28:21,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:28:21,781 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:28:22,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:28:22,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:22,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:23,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:26,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:27,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:32,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:33,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:36,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:36,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:28:36,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:28:39,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:41,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:42,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:45,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:50,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:51,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:28:51,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:28:51,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:28:51,503 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:28:52,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:28:55,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:28:56,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:01,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:02,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:29:06,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:06,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:29:06,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:29:06,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:10,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:29:11,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:11,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:16,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:21,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:29:21,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:21,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:29:21,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:29:21,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:29:22,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:26,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:27,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:31,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:29:33,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:37,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:37,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:29:37,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:29:38,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:41,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:29:43,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:43,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:48,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:51,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:29:51,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:29:52,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:29:52,625 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:29:54,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:58,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:29:59,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:29:59,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:30:04,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:07,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:30:07,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:30:07,627 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:30:09,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:30:10,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:13,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:30:15,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:20,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:21,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:30:21,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:30:22,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:30:22,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:30:25,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:27,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:30:31,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:31,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:30:36,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:37,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:30:37,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:30:37,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:30:41,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:30:41,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:42,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:30:46,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:51,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:30:51,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:30:52,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:52,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:30:52,626 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:30:57,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:30:57,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:00,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:02,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:07,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:07,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:31:07,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:31:08,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:10,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:12,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:14,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:19,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:20,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:21,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:31:22,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:22,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:31:22,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:31:24,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:27,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:30,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:31,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:35,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:37,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:37,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:31:37,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:31:38,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:31:38,880 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:31:38,881 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:31:38,884 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:31:39,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:31:40,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:40,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:41,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:42,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:45,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:51,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:31:51,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:31:51,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:52,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:52,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:31:52,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:31:56,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:31:57,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:31:59,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:32:02,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:07,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:32:07,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:32:09,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:32:12,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:12,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:17,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:19,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:32:21,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:32:22,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:22,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:32:22,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:32:23,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:27,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:29,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:29,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:32:34,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:37,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:37,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:32:37,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:32:39,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:32:39,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:42,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:44,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:49,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:32:50,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:51,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:32:52,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:52,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:32:52,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:32:55,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:32:57,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:32:59,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:01,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:06,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:07,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:33:07,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:33:09,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:11,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:12,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:33:16,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:19,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:21,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:33:22,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:22,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:33:22,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:33:22,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:33:27,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:27,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:33:29,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:32,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:37,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:33:37,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:33:37,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:38,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:42,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:33:43,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:47,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:48,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:51,411 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:33:52,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:33:52,673 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:33:54,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:33:57,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:33:57,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:33:59,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:04,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:07,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:07,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:34:07,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:07,674 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:34:10,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:13,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:16,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:17,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:21,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:21,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:34:23,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:23,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:34:23,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:34:25,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:26,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:28,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:31,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:35,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:37,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:38,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:34:38,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:38,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:34:42,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:43,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:45,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:47,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:51,415 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:34:53,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:53,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:34:53,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:34:53,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:34:53,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:34:53,710 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:34:53,711 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:34:53,713 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:34:54,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:34:55,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:56,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:34:58,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:34:58,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:35:04,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:05,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:35:08,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:35:08,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:35:09,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:13,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:35:15,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:15,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:35:20,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:21,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:35:23,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:35:23,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:35:25,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:25,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:35:28,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:35:30,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:35,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:35:36,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:38,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:35:38,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:35:38,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:35:41,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:43,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:35:46,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:51,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:35:52,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:53,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:35:53,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:35:57,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:35:58,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:36:03,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:04,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:36:08,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:09,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:36:09,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:36:09,713 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:36:13,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:14,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:36:14,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:36:18,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:21,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:36:21,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:36:24,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:24,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:36:24,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:36:24,713 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:36:29,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:30,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:36:31,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:36:35,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:39,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:36:39,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:36:40,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:41,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:36:45,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:36:45,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:50,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:51,429 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:36:54,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:36:54,714 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:36:55,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:36:57,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:37:00,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:00,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:05,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:09,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:37:09,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:37:09,713 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:37:10,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:15,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:15,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:20,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:21,418 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:37:25,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:25,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:37:25,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:37:26,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:27,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:37:31,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:31,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:37,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:37,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:37:40,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:40,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:37:40,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:37:42,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:46,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:47,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:37:48,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:51,419 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:37:54,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:37:55,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:37:55,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:37:55,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:37:57,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:37:59,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:01,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:04,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:07,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:38:10,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:10,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:38:10,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:38:10,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:16,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:16,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:18,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:38:21,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:21,420 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:38:25,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:25,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:38:25,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:38:26,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:38:26,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:31,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:31,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:36,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:38:36,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:40,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:40,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:38:40,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:38:42,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:43,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:38:43,972 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:38:43,973 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:38:43,976 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:38:44,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:38:46,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:38:46,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:47,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:51,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:38:53,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:38:55,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:38:55,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:38:55,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:38:56,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:38:58,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:01,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:39:04,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:04,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:39:09,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:11,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:39:11,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:39:11,556 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:39:13,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:39:14,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:17,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:39:19,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:21,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:39:23,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:39:25,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:26,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:39:26,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:39:30,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:32,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:39:33,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:39:36,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:41,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:39:41,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:39:41,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:39:41,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:43,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:39:47,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:47,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:39:51,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:39:52,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:39:53,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:39:56,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:39:56,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:39:57,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:02,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:40:03,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:03,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:40:08,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:11,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:40:11,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:40:13,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:13,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:40:17,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:40:18,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:21,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:40:21,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:40:24,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:26,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:40:26,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:40:29,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:32,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:40:32,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:40:35,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:40,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:41,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:40:41,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:40:42,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:40:45,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:47,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:40:51,424 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:40:51,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:52,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:40:56,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:40:56,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:40:56,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:41:01,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:02,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:02,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:41:06,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:11,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:41:11,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:41:11,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:12,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:16,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:17,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:41:20,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:21,424 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:41:22,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:27,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:27,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:41:27,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:41:27,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:41:30,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:32,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:33,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:41:37,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:40,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:42,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:41:42,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:41:43,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:48,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:48,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:41:50,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:51,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:41:54,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:41:57,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:41:57,590 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:41:57,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:41:57,594 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:41:57,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:41:57,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:41:57,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:41:58,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:41:59,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:00,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:03,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:42:05,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:07,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:11,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:12,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:42:12,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:42:16,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:17,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:18,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:42:21,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:42:21,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:26,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:27,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:27,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:42:27,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:42:27,917 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:42:32,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:33,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:42:37,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:37,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:42,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:42,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:42:42,913 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:42:47,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:48,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:48,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:42:51,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:42:53,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:42:57,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:42:57,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:42:57,914 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:42:58,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:42:59,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:03,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:04,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:06,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:43:09,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:13,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:13,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:43:13,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:43:14,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:16,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:43:18,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:20,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:21,426 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:43:26,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:26,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:43:28,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:43:28,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:43:31,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:33,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:36,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:43:37,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:42,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:43,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:43,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:43:43,757 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:43:46,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:43:47,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:48,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:51,426 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:43:53,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:56,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:43:58,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:43:58,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:43:58,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:43:58,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:44:03,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:44:03,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:06,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:44:08,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:13,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:44:13,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:44:14,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:16,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:44:18,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:44:19,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:21,429 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:44:24,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:26,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:44:28,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:44:28,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:44:29,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:33,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:44:34,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:44:34,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:39,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:43,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:44:43,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:44:44,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:44:45,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:48,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:44:50,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:51,430 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:44:53,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:44:56,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:44:58,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:44:58,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:45:01,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:03,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:45:04,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:06,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:11,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:12,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:45:12,469 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:45:12,469 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:45:12,471 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:45:13,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:45:13,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:45:13,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:45:14,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:15,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:17,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:18,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:45:21,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:45:23,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:24,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:28,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:45:28,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:45:28,760 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:45:28,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:32,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:33,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:45:34,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:39,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:42,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:43,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:45:43,757 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:45:44,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:49,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:45:49,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:51,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:45:52,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:45:55,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:45:59,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:45:59,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:45:59,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:46:00,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:02,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:46:04,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:46:05,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:11,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:12,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:46:14,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:46:14,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:46:14,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:46:16,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:19,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:46:21,432 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:46:22,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:22,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:46:27,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:29,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:46:29,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:46:29,499 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:46:32,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:46:32,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:34,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:46:37,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:40,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:46:43,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:44,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:46:44,497 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:46:48,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:49,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:46:50,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:46:51,432 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:46:54,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:46:59,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:46:59,497 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:46:59,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:00,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:04,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:04,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:47:10,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:10,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:14,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:47:14,497 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:47:15,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:18,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:19,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:47:21,433 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:47:21,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:26,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:29,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:29,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:47:29,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:47:31,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:36,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:47:36,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:38,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:42,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:46,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:47:46,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:47:46,022 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:47:48,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:48,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:51,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:47:51,434 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:47:53,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:47:58,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:47:58,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:01,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:48:01,020 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:48:04,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:06,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:08,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:09,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:14,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:16,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:16,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:48:16,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:48:18,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:20,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:21,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:21,434 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:48:25,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:25,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:48:25,721 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:48:25,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:48:25,724 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:48:26,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:48:26,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:27,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:30,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:31,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:48:31,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:31,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:48:36,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:36,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:36,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:41,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:46,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:46,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:48:46,023 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:48:46,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:47,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:51,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:48:51,438 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:48:52,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:48:56,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:48:57,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:01,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:49:01,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:49:03,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:06,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:49:06,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:49:08,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:13,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:14,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:49:16,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:49:16,021 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:49:19,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:21,438 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:49:22,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:49:24,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:24,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:49:29,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:49:32,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:49:32,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:49:34,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:34,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:49:38,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:49:39,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:44,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:49:45,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:47,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:49:47,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:49:47,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:49:50,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:49:51,440 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:49:53,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:49:55,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:49:55,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:00,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:02,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:02,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:50:02,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:50:05,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:50:06,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:08,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:11,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:15,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:50:17,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:17,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:50:17,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:50:17,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:21,440 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:50:22,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:23,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:23,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:50:27,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:32,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:32,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:50:32,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:50:33,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:50:33,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:38,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:38,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:42,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:50:43,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:47,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:47,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:50:47,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:50:49,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:50:51,441 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:50:52,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:50:53,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:50:55,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:01,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:02,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:02,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:51:02,475 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:51:02,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:06,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:08,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:10,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:12,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:17,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:17,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:17,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:51:17,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:51:21,442 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:51:22,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:23,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:26,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:27,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:32,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:33,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:33,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:51:33,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:51:36,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:37,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:38,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:42,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:51:44,566 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:51:44,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:51:44,570 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:51:44,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:51:45,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:46,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:48,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:48,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:51:48,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:51:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:51,444 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:51:53,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:51:53,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:51:54,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:51:59,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:03,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:03,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:52:03,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:52:04,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:04,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:52:08,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:09,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:14,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:14,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:52:18,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:18,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:52:18,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:52:20,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:21,445 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:52:23,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:25,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:52:26,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:31,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:33,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:33,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:52:33,160 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:52:35,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:52:36,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:38,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:41,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:45,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:52:46,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:48,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:48,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:52:48,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:52:51,446 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:52:51,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:52:53,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:52:53,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:52:56,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:03,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:03,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:53:03,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:53:03,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:53:08,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:08,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:13,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:53:14,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:19,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:19,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:19,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:53:19,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:53:21,446 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:53:23,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:53:24,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:29,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:30,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:53:34,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:53:34,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:34,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:53:34,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:39,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:39,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:40,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:53:44,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:49,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:49,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:53:49,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:53:50,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:53:50,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:53:51,446 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:53:54,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:53:56,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:00,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:54:01,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:04,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:54:04,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:54:04,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:54:06,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:09,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:54:10,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:54:12,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:17,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:19,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:54:19,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:54:20,962 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:54:21,447 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:54:22,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:24,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:54:27,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:31,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:54:32,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:34,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:54:34,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:54:34,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:54:37,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:39,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:54:39,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:54:43,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:48,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:49,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:54:49,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:54:49,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:54:51,447 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:54:54,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:54:56,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:54:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:54:57,338 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:54:57,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:54:57,342 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:54:58,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:54:59,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:00,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:05,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:05,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:55:05,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:55:05,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:55:09,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:10,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:11,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:55:15,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:19,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:20,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:55:20,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:55:20,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:21,448 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:55:26,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:55:26,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:27,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:31,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:35,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:55:35,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:55:35,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:55:36,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:37,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:41,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:55:41,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:47,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:47,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:50,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:55:50,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:55:51,448 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:55:52,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:55:56,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:55:57,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:55:57,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:02,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:05,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:56:05,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:56:05,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:56:07,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:12,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:56:12,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:14,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:56:18,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:20,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:56:20,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:56:21,450 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:56:23,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:24,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:56:26,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:56:29,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:34,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:34,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:56:35,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:56:35,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:56:39,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:41,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:56:44,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:56:45,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:50,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:50,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:56:50,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:56:50,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:56:51,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:56:55,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:56:55,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:56:56,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:00,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:05,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:57:05,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:57:05,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:57:05,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:10,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:11,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:15,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:57:16,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:21,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:21,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:21,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:57:21,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:57:21,537 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:57:23,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:57:26,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:27,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:31,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:33,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:57:36,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:36,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:57:36,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:57:36,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:41,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:41,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:43,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:57:46,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:51,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:57:51,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:57:51,520 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:57:52,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:57:53,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:57:56,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:57:57,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:03,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:03,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:06,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:58:06,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:58:08,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:10,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 07:58:10,766 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 07:58:10,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 07:58:10,769 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 07:58:11,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:58:11,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 07:58:11,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:13,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:13,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:18,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:21,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:58:21,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:58:21,469 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:58:21,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:24,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:26,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:58:29,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:31,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:34,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:36,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:58:36,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:58:40,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:41,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:58:41,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:45,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:51,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:58:51,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:58:51,536 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:58:51,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:58:51,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:58:56,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:58:56,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:00,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:59:02,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:06,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:59:06,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:06,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:59:07,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:11,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:59:11,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:13,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:18,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:21,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:59:21,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:59:21,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:59:21,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:59:23,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:27,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:28,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:29,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:59:33,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:37,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:37,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:59:37,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:59:39,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:59:39,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:42,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:44,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:49,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 07:59:49,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:51,455 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 07:59:52,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:52,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 07:59:52,270 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 07:59:55,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 07:59:58,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 07:59:59,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:00:01,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:06,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:07,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:00:07,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:00:09,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:00:12,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:12,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:17,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:19,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:00:21,455 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:00:22,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:22,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:00:22,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:00:22,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:27,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:29,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:00:33,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:37,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:37,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:00:37,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:00:38,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:39,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:00:42,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:43,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:48,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:49,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:00:51,456 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:00:52,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:52,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:00:52,271 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:00:53,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:57,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:00:59,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:00:59,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:04,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:01:07,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:01:09,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:09,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:12,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:14,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:20,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:21,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:01:22,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:22,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:01:22,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:01:25,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:27,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:01:27,615 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:01:27,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:01:27,618 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:01:27,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:28,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:01:29,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:30,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:30,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:35,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:37,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:37,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:01:37,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:01:39,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:41,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:42,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:46,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:47,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:51,457 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:01:52,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:52,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:01:52,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:01:52,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:01:57,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:01:57,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:01:58,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:03,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:07,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:07,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:02:07,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:02:07,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:02:08,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:12,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:13,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:17,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:02:18,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:21,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:02:22,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:22,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:02:22,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:02:24,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:27,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:02:27,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:29,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:34,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:37,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:37,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:02:37,272 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:02:37,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:02:40,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:44,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:45,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:47,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:02:50,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:51,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:02:52,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:02:52,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:02:52,275 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:02:56,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:02:57,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:02:57,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:03:01,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:05,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:03:06,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:07,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:03:07,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:03:07,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:03:12,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:13,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:03:15,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:03:17,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:21,461 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:03:23,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:03:23,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:03:23,442 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:03:23,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:25,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:03:28,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:28,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:03:33,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:36,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:03:38,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:03:38,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:03:39,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:43,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:03:44,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:46,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:03:49,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:51,461 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:03:53,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:03:53,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:03:54,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:03:55,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:03:58,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:04:00,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:04,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:06,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:08,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:04:08,441 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:04:11,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:13,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:04:14,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:17,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:21,462 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:04:22,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:23,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:23,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:04:23,442 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:04:28,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:28,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:04:33,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:33,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:38,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:04:38,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:04:38,443 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:04:39,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:41,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:04:41,828 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:04:41,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:04:41,830 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:04:42,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:04:43,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:43,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:04:44,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:44,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:49,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:51,462 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:04:53,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:04:53,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:04:53,443 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:04:53,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:04:55,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:04:59,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:05:00,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:01,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:05:05,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:09,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:05:09,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:05:09,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:05:11,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:11,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:05:14,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:05:16,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:21,463 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:05:21,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:05:22,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:24,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:05:24,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:05:24,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:05:27,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:29,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:05:32,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:35,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:05:38,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:39,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:05:39,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:05:43,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:45,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:05:45,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:05:49,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:51,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:05:54,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:05:54,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:05:54,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:05:56,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:05:59,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:00,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:06:05,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:06,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:06:09,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:06:09,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:06:10,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:15,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:06:16,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:06:16,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:21,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:06:22,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:24,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:06:24,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:06:24,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:06:27,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:31,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:06:33,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:34,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:06:38,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:41,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:06:41,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:06:41,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:06:44,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:44,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:06:47,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:06:49,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:51,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:06:54,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:06:54,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:06:56,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:06:56,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:06:56,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:07:00,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:01,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:07:03,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:07:05,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:10,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:11,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:07:11,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:07:11,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:07:16,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:17,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:07:21,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:07:21,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:21,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:07:26,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:07:26,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:07:26,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:31,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:07:31,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:07:32,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:37,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:41,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:07:41,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:07:41,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:07:42,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:46,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:07:47,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:49,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:07:51,468 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:07:52,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:57,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:07:57,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:07:57,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:07:57,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:07:58,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:07:58,985 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:07:58,986 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:07:58,989 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:07:59,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:07:59,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:00,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:02,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:03,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:08:07,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:09,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:12,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:08:12,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:08:12,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:08:13,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:17,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:08:19,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:20,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:21,468 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:08:25,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:27,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:08:27,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:08:27,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:08:30,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:31,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:33,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:08:36,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:40,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:41,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:42,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:08:42,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:08:46,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:47,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:08:48,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:08:51,468 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:08:52,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:57,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:08:57,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:08:57,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:08:58,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:02,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:03,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:09:08,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:08,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:12,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:09:12,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:09:13,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:17,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:09:18,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:19,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:21,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:09:24,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:26,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:27,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:09:27,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:09:29,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:33,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:09:34,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:36,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:39,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:43,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:09:43,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:09:43,660 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:09:45,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:46,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:48,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:09:50,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:51,469 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:09:55,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:09:55,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:09:58,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:09:58,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:10:01,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:04,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:10:05,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:10:07,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:12,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:13,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:10:13,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:10:13,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:10:15,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:10:17,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:18,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:10:21,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:10:23,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:23,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:10:28,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:28,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:10:28,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:10:33,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:10:34,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:10:34,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:39,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:43,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:10:43,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:10:44,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:10:44,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:48,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:10:49,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:51,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:10:55,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:10:58,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:10:58,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:11:00,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:04,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:11:05,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:06,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:11:10,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:13,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:11:13,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:11:13,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:11:16,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:16,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:11:18,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:11:21,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:21,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:11:23,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:11:23,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:11:23,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:11:23,679 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:11:24,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:11:24,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:11:26,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:28,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:11:28,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:11:31,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:34,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:11:34,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:11:37,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:42,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:43,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:11:43,658 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:11:44,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:11:47,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:48,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:11:51,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:11:53,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:11:54,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:11:58,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:11:58,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:11:58,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:11:58,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:04,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:12:04,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:04,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:12:09,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:12,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:12:13,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:12:13,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:12:14,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:19,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:20,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:12:21,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:12:21,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:12:25,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:29,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:12:29,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:12:29,764 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:12:30,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:32,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:12:35,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:12:36,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:41,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:42,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:12:44,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:12:44,762 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:12:46,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:50,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:12:50,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:12:51,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:12:52,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:57,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:12:59,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:12:59,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:12:59,762 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:13:00,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:13:03,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:05,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:13:08,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:12,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:13:13,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:14,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:13:14,762 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:13:18,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:20,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:13:20,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:13:21,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:13:24,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:29,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:13:29,762 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:13:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:30,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:13:34,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:35,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:13:40,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:40,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:13:44,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:13:44,762 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:13:45,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:49,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:13:50,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:13:51,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:51,477 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:13:56,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:13:59,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:13:59,762 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:14:00,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:14:01,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:04,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:14:06,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:08,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:14:11,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:14,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:14:14,763 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:14:17,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:18,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:14:19,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:14:21,477 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:14:22,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:27,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:28,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:14:29,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:14:29,763 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:14:32,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:35,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:14:37,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:14:37,083 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:14:37,084 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:14:37,086 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:14:37,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:14:38,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:38,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:14:43,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:45,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:14:45,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:14:45,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:14:48,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:49,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:14:50,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:14:51,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:14:53,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:58,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:14:59,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:15:00,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:15:03,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:05,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:06,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:09,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:14,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:15,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:15,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:15:15,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:15:16,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:20,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:20,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:21,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:15:26,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:26,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:15:30,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:15:31,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:36,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:36,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:37,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:42,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:45,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:45,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:15:45,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:15:46,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:47,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:51,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:15:51,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:15:53,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:15:56,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:15:58,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:00,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:16:00,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:16:03,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:06,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:06,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:16:09,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:14,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:16:14,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:15,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:15,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:16:15,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:16:19,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:21,482 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:16:22,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:24,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:16:25,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:30,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:31,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:31,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:16:31,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:16:34,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:16:35,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:36,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:40,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:42,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:16:45,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:46,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:46,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:16:46,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:16:51,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:16:51,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:16:51,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:16:52,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:16:56,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:01,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:01,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:17:01,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:17:02,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:02,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:06,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:07,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:12,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:13,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:16,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:16,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:17:16,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:17:18,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:21,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:21,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:17:22,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:24,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:29,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:31,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:31,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:31,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:17:31,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:17:34,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:37,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:39,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:41,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:44,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:46,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:46,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:17:46,398 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:17:49,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:17:49,466 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:17:49,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:17:49,470 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:17:50,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:17:50,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:50,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:17:51,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:17:51,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:17:52,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:17:55,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:00,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:01,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:01,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:01,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:18:01,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:18:06,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:07,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:10,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:11,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:16,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:17,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:17,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:18:17,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:18:20,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:21,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:18:22,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:23,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:27,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:28,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:32,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:32,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:18:32,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:18:33,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:37,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:38,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:38,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:43,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:47,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:47,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:18:47,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:18:48,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:49,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:51,484 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:18:53,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:18:54,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:18:58,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:18:59,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:02,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:02,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:19:02,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:19:05,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:06,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:19:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:11,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:16,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:17,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:17,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:19:17,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:19:18,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:19:21,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:21,484 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:19:23,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:27,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:19:27,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:32,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:32,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:19:32,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:19:32,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:37,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:19:38,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:38,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:43,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:47,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:19:47,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:47,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:19:47,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:19:48,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:51,485 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:19:53,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:19:53,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:19:57,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:19:58,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:02,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:02,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:20:02,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:20:04,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:08,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:09,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:11,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:20:14,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:18,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:18,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:20:18,449 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:20:19,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:21,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:20:21,485 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:20:24,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:25,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:31,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:31,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:20:33,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:33,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:20:33,448 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:20:36,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:38,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:41,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:20:42,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:47,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:48,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:48,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:20:48,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:20:50,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:20:51,486 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:20:53,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:20:54,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:20:58,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:00,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:03,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:03,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:21:03,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:21:03,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:08,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:09,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:10,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:21:10,348 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:21:10,348 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:21:10,351 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:21:10,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:21:10,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:11,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:14,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:18,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:21:18,753 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:21:19,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:20,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:21,486 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:21:24,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:25,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:30,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:30,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:33,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:33,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:21:33,753 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:21:35,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:39,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:41,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:41,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:46,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:48,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:48,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:21:48,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:21:51,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:21:51,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:21:52,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:21:54,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:21:57,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:01,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:22:03,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:03,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:03,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:22:03,754 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:22:08,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:09,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:11,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:22:13,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:22:18,754 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:22:18,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:19,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:22:21,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:22:24,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:24,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:29,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:22:30,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:33,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:33,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:22:33,754 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:22:35,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:38,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:39,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:22:41,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:46,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:48,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:48,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:22:48,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:22:49,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:22:51,488 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:22:51,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:53,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:22:56,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:22:59,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:01,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:05,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:23:05,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:23:05,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:23:07,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:09,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:12,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:17,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:17,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:20,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:23:20,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:23:20,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:23:21,489 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:23:23,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:25,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:23:27,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:28,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:33,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:35,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:23:35,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:23:37,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:39,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:44,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:46,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:49,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:23:50,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:23:50,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:23:51,491 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:23:54,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:23:55,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:00,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:05,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:24:05,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:24:05,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:24:05,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:10,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:15,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:24:16,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:21,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:21,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:24:21,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:24:21,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:24:21,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:24:21,589 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:24:22,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:24:22,427 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:24:22,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:24:22,430 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:24:23,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:24:23,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:24:25,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:24:26,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:31,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:36,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:24:36,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:24:36,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:24:36,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:24:36,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:41,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:46,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:51,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:24:51,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:24:51,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:24:51,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:24:51,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:24:51,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:56,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:24:57,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:25:01,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:06,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:25:06,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:25:06,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:25:06,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:25:07,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:25:07,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:12,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:17,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:25:18,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:21,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:25:21,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:25:21,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:25:21,593 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:25:23,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:27,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:25:29,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:34,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:36,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:25:36,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:25:36,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:25:37,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:25:39,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:44,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:47,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:25:49,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:51,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:25:51,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:25:51,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:25:51,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:25:55,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:25:57,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:00,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:06,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:06,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:26:06,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:26:06,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:26:06,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:26:07,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:11,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:15,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:17,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:21,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:26:21,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:26:21,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:26:21,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:26:22,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:25,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:28,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:33,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:35,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:36,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:26:36,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:26:38,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:43,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:45,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:48,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:51,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:26:52,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:26:52,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:26:52,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:26:52,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:26:54,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:26:55,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:26:59,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:05,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:27:07,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:27:07,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:27:07,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:27:10,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:13,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:27:16,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:21,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:27:21,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:22,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:27:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:27:22,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:27:23,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:27:27,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:32,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:33,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:27:37,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:27:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:27:37,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:27:38,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:43,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:27:43,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:48,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:51,498 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:27:52,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:27:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:27:52,327 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:27:53,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:27:54,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:27:59,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:01,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:28:01,538 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:28:01,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:28:01,543 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:28:01,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:28:03,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:28:04,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:07,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:28:07,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:28:07,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:28:10,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:13,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:28:16,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:21,498 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:28:21,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:28:22,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:28:22,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:28:23,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:28:26,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:31,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:33,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:28:36,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:28:37,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:28:37,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:28:42,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:28:42,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:47,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:51,499 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:28:52,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:28:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:28:52,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:28:52,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:28:53,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:28:58,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:02,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:29:03,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:08,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:08,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:08,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:29:08,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:29:09,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:11,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:29:14,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:20,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:21,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:29:21,499 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:29:23,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:23,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:23,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:29:23,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:29:25,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:29,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:29:30,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:35,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:38,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:38,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:38,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:29:38,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:29:39,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:29:41,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:46,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:49,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:29:51,500 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:29:52,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:53,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:29:53,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:29:53,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:29:58,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:29:59,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:03,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:07,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:08,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:30:08,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:30:08,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:30:09,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:14,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:17,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:20,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:21,501 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:30:24,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:30:24,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:30:24,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:30:24,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:30:25,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:27,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:31,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:36,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:37,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:39,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:30:39,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:30:39,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:30:41,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:47,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:47,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:51,501 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:30:52,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:30:55,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:30:55,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:30:55,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:30:55,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:30:58,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:30:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:03,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:08,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:31:09,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:10,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:31:10,223 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:31:14,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:15,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:31:15,875 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:31:15,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:31:15,879 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:31:16,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:31:18,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:31:19,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:21,502 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:31:25,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:31:25,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:31:25,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:31:25,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:31:25,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:28,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:31:30,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:35,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:38,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:31:40,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:31:40,223 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:31:41,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:46,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:47,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:31:51,502 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:31:52,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:31:55,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:31:55,225 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:31:57,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:31:58,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:03,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:08,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:10,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:32:10,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:32:13,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:15,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:32:19,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:21,504 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:32:25,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:25,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:32:25,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:32:25,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:32:30,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:35,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:32:35,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:40,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:32:40,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:32:40,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:32:40,228 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:32:41,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:45,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:32:47,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:51,505 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:32:52,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:32:53,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:32:55,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:32:55,224 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:32:58,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:04,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:33:04,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:09,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:10,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:33:10,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:33:10,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:33:14,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:33:14,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:19,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:21,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:33:24,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:33:25,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:33:25,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:33:25,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:33:25,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:30,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:32,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:33:35,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:40,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:33:40,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:33:40,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:33:40,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:33:41,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:42,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:33:46,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:51,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:33:52,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:33:52,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:33:55,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:33:55,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:33:57,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:02,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:34:03,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:08,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:10,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:34:10,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:34:10,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:34:13,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:18,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:20,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:34:21,507 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:34:24,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:25,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:34:25,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:34:30,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:31,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:34:35,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:39,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:34:39,236 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:34:39,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:34:39,239 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:34:39,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:34:39,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:34:40,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:34:40,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:34:40,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:34:40,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:34:40,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:45,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:49,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:34:50,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:51,507 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:34:55,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:34:55,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:34:55,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:34:55,471 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:34:56,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:34:59,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:01,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:06,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:11,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:35:11,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:35:11,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:35:11,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:35:12,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:14,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:17,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:21,509 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:35:23,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:24,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:26,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:35:26,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:35:26,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:35:29,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:34,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:35,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:40,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:35:41,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:35:41,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:35:41,538 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:35:44,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:45,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:50,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:51,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:51,510 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:35:56,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:35:56,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:35:56,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:35:56,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:35:56,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:36:01,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:06,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:36:06,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:12,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:12,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:36:12,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:36:12,601 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:36:16,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:36:17,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:21,512 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:36:23,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:26,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:36:27,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:36:27,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:36:27,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:36:28,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:33,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:34,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:36:38,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:42,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:36:42,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:36:42,599 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:36:44,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:36:44,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:49,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:51,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:36:54,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:36:55,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:36:57,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:36:57,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:36:57,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:37:00,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:04,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:37:06,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:12,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:37:12,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:37:12,601 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:37:13,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:37:17,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:21,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:37:21,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:37:23,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:28,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:29,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:37:29,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:37:29,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:37:31,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:37:34,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:39,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:42,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:37:44,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:37:44,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:37:44,109 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:37:45,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:49,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:37:49,721 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:37:49,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:37:49,724 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:37:50,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:37:50,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:51,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:37:52,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:37:56,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:37:59,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:37:59,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:37:59,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:37:59,110 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:38:00,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:38:02,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:07,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:10,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:38:13,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:14,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:38:14,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:38:14,109 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:38:18,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:20,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:38:21,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:38:23,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:28,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:29,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:38:29,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:38:29,109 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:38:30,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:38:34,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:38,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:38:40,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:45,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:45,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:38:45,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:38:45,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:38:45,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:38:48,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:38:50,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:51,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:38:56,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:38:58,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:00,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:39:00,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:39:00,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:39:00,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:39:01,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:07,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:08,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:12,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:15,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:39:15,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:39:15,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:39:15,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:39:17,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:18,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:21,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:39:23,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:26,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:29,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:30,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:39:30,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:39:34,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:36,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:39,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:45,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:45,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:39:45,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:39:46,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:50,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:39:51,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:39:56,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:39:56,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:00,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:40:00,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:40:01,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:06,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:40:07,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:12,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:15,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:40:15,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:40:16,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:40:18,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:21,517 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:40:23,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:24,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:40:28,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:32,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:40:32,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:40:32,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:40:34,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:34,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:40:39,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:44,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:40:45,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:40:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:40:47,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:40:47,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:40:50,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:40:51,519 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:40:54,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:40:55,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:00,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:41:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:41:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:41:02,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:41:02,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:41:02,500 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:41:02,501 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:41:02,504 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:41:02,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:41:04,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:41:06,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:11,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:14,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:41:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:41:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:41:17,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:41:17,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:21,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:41:23,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:24,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:41:28,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:32,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:41:32,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:41:32,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:41:33,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:34,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:41:38,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:42,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:41:43,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:47,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:41:47,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:41:47,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:41:49,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:51,531 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:41:52,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:41:54,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:41:59,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:42:02,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:42:02,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:42:02,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:42:05,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:11,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:13,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:42:16,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:18,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:42:18,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:42:18,752 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:42:21,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:42:21,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:23,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:42:26,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:31,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:42:31,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:33,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:42:33,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:42:33,752 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:42:37,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:41,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:42:42,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:47,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:48,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:42:48,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:42:48,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:42:48,753 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:42:50,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:42:51,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:42:52,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:42:57,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:00,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:43:02,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:03,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:43:03,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:43:03,752 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:43:07,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:10,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:43:13,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:18,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:43:18,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:43:18,753 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:43:18,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:20,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:43:21,522 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:43:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:29,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:30,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:43:33,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:43:33,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:43:33,753 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:43:35,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:38,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:43:41,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:46,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:48,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:43:48,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:43:48,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:43:48,753 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:43:51,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:43:51,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:56,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:43:58,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:44:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:05,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:44:05,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:44:05,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:44:05,299 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:44:07,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:08,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:44:12,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:15,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:44:15,906 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:44:15,907 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:44:15,911 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:44:15,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:44:16,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:44:17,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:20,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:44:20,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:44:20,298 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:44:21,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:44:23,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:26,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:44:29,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:34,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:35,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:44:35,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:44:35,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:44:35,299 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:44:37,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:44:39,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:44,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:47,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:44:49,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:50,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:44:50,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:44:50,299 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:44:51,534 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:44:55,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:44:57,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:00,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:05,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:45:05,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:45:05,299 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:45:06,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:07,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:11,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:15,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:16,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:20,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:45:20,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:45:20,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:45:20,303 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:45:21,534 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:45:22,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:24,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:28,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:34,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:34,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:35,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:45:35,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:45:35,299 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:45:39,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:44,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:45,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:50,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:45:51,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:45:51,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:45:51,738 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:45:54,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:45:55,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:00,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:02,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:46:06,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:06,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:46:06,478 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:46:11,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:12,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:46:16,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:21,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:46:21,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:46:21,695 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:46:21,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:22,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:46:26,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:31,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:32,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:46:36,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:46:36,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:46:37,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:43,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:46:43,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:48,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:51,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:46:51,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:46:51,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:46:51,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:46:53,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:46:58,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:01,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:47:04,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:06,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:47:06,482 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:47:09,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:11,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:47:14,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:20,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:21,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:47:21,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:47:21,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:47:21,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:47:25,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:28,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:47:28,891 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:47:28,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:47:28,894 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:47:29,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:47:30,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:31,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:47:35,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:37,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:47:37,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:47:37,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:47:39,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:47:41,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:46,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:49,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:47:51,537 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:47:51,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:52,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:47:52,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:47:52,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:47:57,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:47:59,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:48:02,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:07,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:48:07,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:48:07,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:48:07,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:48:07,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:08,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:48:12,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:18,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:18,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:48:21,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:48:22,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:48:22,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:48:22,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:48:23,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:28,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:28,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:48:33,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:36,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:48:37,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:48:37,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:48:37,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:48:39,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:44,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:46,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:48:49,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:51,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:48:52,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:48:52,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:48:52,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:48:55,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:48:57,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:49:00,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:06,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:07,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:49:07,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:49:07,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:49:07,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:49:12,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:17,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:21,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:49:21,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:49:22,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:49:22,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:49:22,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:49:22,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:27,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:31,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:49:33,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:38,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:49:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:49:38,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:49:38,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:49:41,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:49:43,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:48,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:51,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:49:51,570 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:49:53,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:49:53,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:49:53,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:49:53,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:49:53,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:49:58,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:01,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:50:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:08,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:50:08,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:50:08,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:50:08,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:50:09,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:11,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:50:14,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:20,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:21,540 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:50:21,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:50:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:50:23,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:50:23,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:50:25,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:29,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:50:31,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:36,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:50:38,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:50:38,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:50:39,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:50:41,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:46,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:48,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:50:48,242 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:50:48,242 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:50:48,245 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:50:48,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:50:48,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:50:51,540 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:50:52,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:53,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:50:53,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:50:53,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:50:57,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:50:58,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:51:03,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:08,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:51:08,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:51:08,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:51:08,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:09,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:51:13,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:19,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:51:19,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:21,541 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:51:23,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:51:23,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:51:23,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:51:24,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:29,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:51:30,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:35,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:38,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:51:38,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:51:38,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:51:39,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:51:40,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:45,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:47,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:51:51,541 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:51:51,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:54,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:51:54,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:51:54,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:51:57,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:51:57,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:02,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:07,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:07,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:09,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:52:09,991 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:52:13,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:17,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:18,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:21,542 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:52:24,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:24,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:52:24,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:52:24,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:52:24,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:52:27,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:30,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:36,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:37,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:39,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:52:39,991 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:52:42,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:47,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:47,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:51,544 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:52:53,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:52:54,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:52:54,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:52:55,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:52:59,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:04,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:05,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:53:09,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:09,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:53:09,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:53:14,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:15,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:53:19,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:21,545 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:53:24,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:53:24,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:53:25,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:25,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:53:30,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:35,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:36,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:53:40,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:41,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:53:41,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:53:41,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:53:41,534 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:53:43,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:53:45,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:50,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:51,546 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:53:53,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:53:55,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:53:56,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:53:56,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:53:56,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:53:56,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:54:01,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:01,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:54:01,659 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:54:01,660 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:54:01,662 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:54:02,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:54:03,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:54:06,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:11,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:54:11,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:54:11,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:54:11,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:54:11,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:13,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:54:16,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:21,546 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:54:21,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:23,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:54:26,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:54:26,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:54:26,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:54:27,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:33,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:54:33,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:38,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:41,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:54:41,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:54:41,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:54:43,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:54:43,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:48,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:51,546 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:54:53,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:54:54,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:54:56,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:54:56,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:54:56,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:54:59,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:03,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:55:05,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:10,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:11,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:55:11,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:55:11,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:55:11,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:55:15,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:21,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:21,547 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:55:21,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:55:26,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:55:26,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:55:26,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:55:26,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:31,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:31,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:55:36,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:41,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:55:41,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:55:41,531 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:55:42,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:55:42,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:47,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:51,547 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:55:52,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:55:53,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:55:57,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:55:57,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:55:57,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:55:57,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:55:58,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:00,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:56:03,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:09,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:56:09,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:12,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:56:12,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:56:12,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:56:14,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:19,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:56:20,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:21,548 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:56:25,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:27,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:56:27,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:56:27,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:56:27,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:56:29,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:56:31,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:36,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:39,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:56:41,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:42,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:56:42,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:56:46,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:49,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:56:51,548 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:56:52,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:57,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:56:57,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:56:58,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:56:59,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:57:03,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:07,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:57:08,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:12,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:57:12,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:57:14,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:17,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 08:57:17,293 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 08:57:17,294 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 08:57:17,297 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 08:57:17,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 08:57:19,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:19,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:57:21,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:57:24,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:27,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:57:27,834 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:57:27,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:57:30,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:35,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:37,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:57:40,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:42,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:57:42,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:57:46,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:47,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:57:51,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:51,560 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:57:56,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:57:57,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:57:57,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:57:58,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:02,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:07,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:08,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:12,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:12,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:58:12,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:58:12,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:58:12,840 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:58:17,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:18,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:21,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:58:23,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:27,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:58:27,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:58:28,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:29,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:34,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:38,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:39,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:43,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:58:43,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:58:43,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:58:43,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:58:44,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:46,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:49,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:51,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:58:54,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:58:55,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:58:58,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:58:58,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:58:58,714 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:58:59,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:04,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:05,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:59:09,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:13,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 08:59:13,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:59:13,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:59:15,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:15,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:59:20,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:21,551 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:59:25,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:59:26,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:28,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:59:28,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:59:31,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:35,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:59:37,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:42,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:43,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:59:43,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 08:59:43,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:59:47,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:51,553 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 08:59:52,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:53,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 08:59:57,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 08:59:58,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 08:59:58,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:00:03,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:03,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:00:08,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:13,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:00:13,715 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:00:13,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:14,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:00:18,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:21,553 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:00:24,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:00:24,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:30,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:00:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:00:30,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:00:31,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:00:31,553 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:00:31,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:00:31,557 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:00:32,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:00:34,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:00:35,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:40,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:42,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:00:45,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:00:45,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:00:45,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:00:46,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:51,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:00:51,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:00:52,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:00:56,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:01:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:01:00,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:01:01,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:02,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:01:06,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:12,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:01:12,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:15,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:01:15,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:01:15,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:01:17,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:21,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:01:22,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:01:22,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:27,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:01:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:01:30,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:01:32,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:32,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:01:37,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:42,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:01:42,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:45,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:01:45,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:01:45,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:01:48,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:49,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:01:51,555 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:01:53,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:58,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:01:59,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:00,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:02:00,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:02:00,318 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:02:04,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:09,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:09,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:14,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:15,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:02:15,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:02:15,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:02:15,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:02:19,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:21,555 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:02:25,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:30,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:30,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:02:30,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:02:30,319 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:02:31,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:36,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:40,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:42,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:46,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:02:46,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:02:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:02:46,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:02:47,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:50,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:51,556 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:02:53,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:02:58,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:02:59,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:01,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:03:01,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:01,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:03:01,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:04,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:08,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:03:09,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:14,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:03:16,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:03:18,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:03:20,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:21,556 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:03:26,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:31,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:31,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:31,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:31,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:03:31,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:03:32,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:03:36,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:42,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:42,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:03:46,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:03:46,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:03:46,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:03:47,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:50,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:03:50,971 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:03:50,972 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:03:50,974 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:03:51,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:03:51,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:03:51,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:03:52,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:03:53,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:03:58,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:04:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:04:01,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:04:02,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:04:03,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:08,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:12,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:04:14,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:04:16,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:04:16,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:04:19,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:21,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:04:21,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:04:25,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:29,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:04:30,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:31,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:04:31,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:04:31,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:04:35,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:40,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:04:41,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:46,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:47,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:04:47,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:04:47,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:04:47,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:04:50,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:04:51,558 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:04:51,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:04:56,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:00,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:02,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:02,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:05:02,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:05:02,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:05:02,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:05:07,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:10,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:13,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:17,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:05:17,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:05:17,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:05:18,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:18,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:21,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:05:22,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:24,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:28,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:29,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:32,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:05:32,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:05:32,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:05:34,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:38,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:40,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:45,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:47,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:05:47,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:05:47,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:05:48,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:05:50,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:51,561 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:05:56,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:05:58,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:06:02,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:02,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:06:02,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:06:02,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:06:07,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:08,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:06:12,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:17,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:06:17,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:06:17,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:06:17,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:06:17,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:18,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:06:21,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:06:23,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:28,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:06:29,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:32,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:06:32,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:06:32,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:06:32,701 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:06:34,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:36,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:06:40,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:46,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:46,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:06:47,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:06:47,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:06:47,698 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:06:51,560 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:06:51,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:56,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:06:57,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:01,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:03,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:07:03,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:07:03,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:07:04,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:07:04,694 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:07:04,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:07:04,699 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:07:05,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:07:06,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:07,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:07,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:12,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:16,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:18,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:18,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:07:18,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:07:21,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:07:23,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:26,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:29,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:33,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:07:33,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:07:33,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:07:33,491 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:07:35,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:36,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:40,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:44,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:45,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:48,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:07:48,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:07:50,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:07:51,564 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:07:54,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:07:56,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:01,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:03,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:08:03,489 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:08:04,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:08:06,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:11,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:14,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:08:17,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:18,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:08:18,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:08:21,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:08:23,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:24,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:08:28,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:32,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:08:33,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:08:33,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:08:33,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:38,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:42,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:08:44,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:49,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:50,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:08:50,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:08:50,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:08:50,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:08:51,565 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:08:52,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:08:54,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:08:59,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:02,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:09:05,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:09:05,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:09:05,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:09:05,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:10,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:13,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:09:15,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:20,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:09:20,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:09:20,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:09:21,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:21,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:09:23,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:09:26,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:31,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:09:31,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:35,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:09:35,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:09:37,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:41,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:09:42,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:47,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:50,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:09:50,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:09:51,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:09:51,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:09:53,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:09:58,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:00,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:10:03,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:05,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:10:05,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:10:05,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:10:09,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:10,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:10:14,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:18,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:10:18,587 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:10:18,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:10:18,590 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:10:19,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:10:19,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:20,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:10:20,011 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:10:20,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:10:21,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:10:25,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:30,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:10:30,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:35,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:10:35,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:10:36,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:40,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:10:41,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:46,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:50,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:10:50,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:10:50,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:10:50,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:10:51,567 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:10:51,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:10:56,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:00,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:11:02,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:05,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:11:05,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:11:05,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:11:05,015 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:11:07,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:08,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:11:12,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:17,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:18,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:11:20,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:11:20,017 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:11:21,568 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:11:23,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:28,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:29,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:11:33,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:36,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:11:36,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:11:36,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:11:39,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:11:39,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:44,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:49,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:11:49,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:51,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:11:51,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:11:51,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:11:51,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:11:55,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:11:59,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:01,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:06,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:06,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:12:06,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:12:06,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:12:06,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:12:09,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:11,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:17,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:17,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:21,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:12:21,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:12:21,394 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:12:21,582 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:12:22,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:27,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:28,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:33,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:36,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:12:36,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:12:36,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:12:36,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:38,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:43,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:46,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:48,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:51,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:12:51,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:12:51,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:12:51,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:12:51,592 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:12:54,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:12:56,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:12:59,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:04,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:04,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:06,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:13:06,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:13:06,395 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:13:10,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:14,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:16,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:21,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:21,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:13:21,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:13:21,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:13:21,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:13:21,632 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:13:24,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:26,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:31,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:32,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:13:32,087 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:13:32,088 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:13:32,091 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:13:32,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:13:32,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:34,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:36,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:13:36,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:13:36,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:41,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:43,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:47,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:51,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:13:51,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:13:51,628 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:13:52,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:13:53,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:13:57,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:03,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:03,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:14:06,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:14:06,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:14:08,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:11,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:14:14,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:19,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:21,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:14:21,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:14:21,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:14:21,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:14:24,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:29,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:31,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:14:35,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:36,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:14:36,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:14:41,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:41,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:14:46,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:51,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:14:51,574 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:14:51,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:14:51,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:14:53,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:14:56,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:02,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:03,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:15:06,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:15:06,629 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:15:07,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:13,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:13,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:15:18,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:21,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:15:21,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:15:21,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:15:22,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:15:23,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:28,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:32,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:15:34,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:36,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:15:36,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:15:39,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:42,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:15:44,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:49,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:15:51,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:15:51,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:15:51,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:15:52,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:15:55,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:01,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:03,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:16:06,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:06,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:16:06,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:16:11,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:16:12,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:17,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:21,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:16:21,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:16:21,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:16:21,630 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:16:22,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:27,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:31,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:16:33,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:37,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:16:37,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:16:37,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:16:37,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:16:39,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:41,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:16:44,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:48,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:16:48,780 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:16:48,781 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:16:48,783 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:16:49,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:16:49,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:51,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:16:51,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:16:52,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:16:52,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:16:52,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:16:52,874 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:16:55,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:16:59,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:17:00,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:05,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:07,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:17:07,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:17:07,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:17:07,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:17:09,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:17:11,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:16,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:19,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:17:21,577 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:17:21,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:22,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:17:22,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:17:22,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:17:27,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:29,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:17:32,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:37,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:17:37,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:17:37,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:17:38,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:39,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:17:43,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:48,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:51,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:17:52,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:17:52,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:17:52,880 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:17:54,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:17:54,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:17:59,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:04,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:04,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:18:07,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:18:07,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:18:07,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:18:10,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:15,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:18:16,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:21,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:21,580 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:18:22,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:18:22,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:18:22,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:18:22,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:18:25,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:18:26,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:31,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:35,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:18:37,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:37,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:18:37,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:18:37,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:18:42,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:45,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:18:47,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:51,582 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:18:52,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:18:52,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:18:52,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:18:53,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:18:55,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:18:58,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:03,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:05,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:19:07,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:19:07,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:19:07,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:19:09,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:14,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:15,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:19:19,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:21,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:19:22,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:19:22,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:19:22,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:19:23,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:19:25,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:30,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:33,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:19:35,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:37,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:19:37,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:19:37,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:19:40,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:43,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:19:45,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:50,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:19:51,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:19:52,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:19:52,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:19:52,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:19:52,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:19:53,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:19:56,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:01,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:03,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:06,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:07,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:07,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:07,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:20:07,878 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:20:11,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:11,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:20:11,520 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:20:11,521 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:20:11,523 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:20:11,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:20:12,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:13,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:16,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:21,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:20:21,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:22,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:22,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:22,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:20:22,878 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:20:23,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:27,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:32,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:33,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:37,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:39,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:39,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:39,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:20:39,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:20:42,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:43,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:48,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:51,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:20:51,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:20:53,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:20:54,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:20:54,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:20:54,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:20:59,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:01,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:21:04,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:09,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:21:09,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:21:09,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:21:10,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:11,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:21:15,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:20,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:21,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:21:21,585 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:21:24,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:21:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:21:24,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:21:26,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:31,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:21:31,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:36,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:39,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:21:39,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:21:39,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:21:41,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:21:42,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:47,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:51,585 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:21:51,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:21:52,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:54,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:21:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:21:54,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:21:58,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:21:59,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:22:03,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:09,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:22:09,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:22:09,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:22:09,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:09,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:22:14,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:19,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:19,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:22:21,585 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:22:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:22:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:22:24,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:22:25,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:29,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:22:31,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:36,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:39,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:22:39,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:22:39,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:22:39,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:22:41,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:46,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:50,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:22:51,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:22:51,586 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:22:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:22:54,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:22:54,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:22:57,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:02,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:07,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:09,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:23:09,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:23:09,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:23:13,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:18,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:20,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:23:21,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:23:24,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:23:24,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:23:24,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:23:24,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:29,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:30,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:23:34,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:39,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:23:39,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:23:39,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:23:39,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:23:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:45,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:48,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:23:48,799 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:23:48,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:23:48,802 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:23:49,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:23:49,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:23:50,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:51,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:23:51,588 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:23:54,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:23:54,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:23:54,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:23:56,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:23:59,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:24:02,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:07,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:09,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:24:09,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:24:09,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:24:09,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:24:12,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:18,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:19,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:24:21,588 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:24:23,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:24,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:24:24,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:24:24,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:24:29,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:29,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:24:34,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:39,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:24:39,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:24:39,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:24:39,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:39,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:24:44,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:47,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:24:49,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:51,589 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:24:55,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:24:55,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:24:55,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:24:55,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:24:55,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:24:57,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:25:01,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:07,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:08,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:25:10,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:25:10,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:25:10,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:25:10,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:25:12,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:18,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:25:18,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:21,589 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:25:23,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:25,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:25:25,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:25:25,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:25:25,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:25:28,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:25:28,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:33,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:38,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:25:39,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:40,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:25:40,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:25:40,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:25:44,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:48,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:25:49,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:51,590 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:25:55,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:25:55,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:25:55,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:25:55,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:25:58,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:26:00,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:06,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:08,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:26:10,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:26:10,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:26:10,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:26:11,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:17,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:18,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:26:21,590 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:26:23,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:25,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:26:25,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:26:25,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:26:27,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:26:28,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:33,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:37,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:26:39,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:40,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:26:40,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:26:40,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:26:44,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:49,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:51,592 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:26:55,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:26:55,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:26:55,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:26:55,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:26:55,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:00,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:05,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:06,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:10,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:27:10,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:27:10,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:27:11,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:14,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:27:14,045 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:27:14,046 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:27:14,048 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:27:14,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:27:15,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:17,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:21,594 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:27:22,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:25,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:27:25,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:27:25,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:27:25,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:27,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:32,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:36,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:38,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:40,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:27:40,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:27:40,510 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:27:43,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:46,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:48,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:51,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:27:54,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:27:55,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:27:55,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:27:55,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:27:56,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:27:59,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:05,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:06,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:28:10,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:10,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:28:10,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:28:10,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:28:15,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:16,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:28:20,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:21,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:28:25,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:28:25,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:28:25,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:28:26,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:28:26,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:31,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:34,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:28:36,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:41,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:28:41,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:28:41,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:28:41,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:28:42,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:44,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:28:47,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:51,597 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:28:53,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:28:54,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:28:56,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:28:56,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:28:56,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:28:58,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:03,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:29:04,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:09,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:11,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:29:11,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:29:11,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:29:11,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:29:13,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:29:14,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:19,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:21,597 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:29:23,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:29:24,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:26,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:29:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:29:26,371 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:29:30,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:34,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:29:36,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:41,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:29:41,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:29:41,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:29:41,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:29:41,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:44,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:29:46,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:51,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:29:51,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:29:54,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:29:56,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:29:56,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:29:56,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:29:57,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:03,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:04,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:08,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:11,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:30:11,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:30:11,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:30:12,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:14,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:19,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:21,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:30:22,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:24,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:30:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:30:26,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:30:30,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:30,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:30:30,587 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:30:30,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:30:30,590 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:30:31,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:30:31,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:32,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:40,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:41,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:30:41,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:30:41,372 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:30:42,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:45,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:50,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:51,599 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:30:54,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:30:56,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:30:56,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:30:56,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:30:56,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:31:01,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:04,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:31:06,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:11,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:31:11,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:31:11,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:31:11,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:31:11,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:13,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:31:16,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:21,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:31:21,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:31:22,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:26,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:31:26,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:31:26,373 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:31:28,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:32,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:31:34,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:39,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:41,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:31:41,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:31:41,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:31:41,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:31:44,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:31:44,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:49,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:51,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:31:54,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:31:54,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:31:56,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:31:56,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:31:56,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:31:59,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:02,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:32:05,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:10,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:11,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:32:11,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:32:11,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:32:12,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:32:15,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:20,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:21,601 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:32:22,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:32:26,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:32:26,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:32:26,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:32:26,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:32,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:32,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:32:37,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:41,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:32:41,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:32:41,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:32:42,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:44,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:32:47,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:51,601 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:32:52,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:32:53,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:32:56,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:32:56,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:32:56,404 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:32:58,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:03,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:06,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:33:09,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:11,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:33:11,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:33:11,374 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:33:14,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:16,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:33:20,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:21,604 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:33:25,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:26,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:33:26,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:33:26,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:33:26,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:33:31,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:36,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:33:37,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:41,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:33:41,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:33:41,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:33:42,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:47,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:33:47,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:51,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:33:53,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:33:54,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:33:54,667 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:33:54,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:33:54,669 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:33:55,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:33:56,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:33:56,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:33:56,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:33:56,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:33:59,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:04,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:09,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:11,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:34:11,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:34:11,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:34:14,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:19,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:21,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:34:25,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:26,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:34:26,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:34:26,375 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:34:30,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:35,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:40,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:41,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:34:41,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:34:41,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:34:46,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:51,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:34:51,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:34:56,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:34:56,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:34:56,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:34:57,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:02,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:07,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:11,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:35:11,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:35:11,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:35:12,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:17,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:21,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:35:23,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:26,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:35:26,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:35:26,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:35:28,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:33,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:38,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:41,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:35:41,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:35:41,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:35:43,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:48,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:51,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:35:54,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:35:56,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:35:56,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:35:56,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:35:59,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:04,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:09,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:11,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:36:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:36:11,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:36:15,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:20,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:21,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:36:26,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:36:26,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:36:26,376 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:36:26,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:31,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:36,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:41,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:36:41,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:36:41,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:36:41,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:46,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:51,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:36:51,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:36:56,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:36:56,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:36:56,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:36:57,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:02,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:07,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:37:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:37:11,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:37:13,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:18,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:21,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:37:23,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:26,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:37:26,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:37:26,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:37:29,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:34,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:39,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:41,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:37:41,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:37:41,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:37:44,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:49,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:51,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:37:55,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:37:56,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:37:56,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:37:56,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:38:00,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:05,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:10,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:38:11,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:38:11,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:38:16,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:21,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:21,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:38:26,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:38:26,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:38:26,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:38:26,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:38:26,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:31,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:36,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:41,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:38:41,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:38:41,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:38:42,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:47,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:51,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:38:53,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:38:56,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:38:56,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:38:56,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:38:58,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:03,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:08,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:11,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:39:11,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:39:11,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:39:14,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:19,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:21,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:39:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:26,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:39:26,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:39:26,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:39:26,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:39:30,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:35,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:41,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:39:41,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:39:41,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:39:45,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:50,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:39:51,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:39:56,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:39:56,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:39:56,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:01,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:06,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:11,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:40:11,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:40:11,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:40:12,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:17,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:21,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:40:22,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:26,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:40:26,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:40:26,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:40:28,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:33,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:38,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:41,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:40:41,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:40:43,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:48,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:51,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:40:54,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:40:56,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:40:56,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:40:59,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:04,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:09,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:11,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:41:11,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:41:11,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:41:15,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:20,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:21,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:41:25,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:26,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:41:26,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:41:31,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:36,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:41,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:41:41,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:41:42,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:47,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:51,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:41:52,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:41:56,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:41:56,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:41:56,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:41:58,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:03,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:08,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:11,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:42:11,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:42:13,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:18,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:21,614 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:42:24,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:26,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:42:26,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:42:29,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:34,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:39,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:41,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:42:41,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:42:45,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:50,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:51,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:42:55,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:42:56,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:42:56,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:43:00,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:05,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:10,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:11,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:43:11,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:43:16,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:21,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:43:22,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:26,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:43:26,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:43:28,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:33,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:38,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:41,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:43:41,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:43:43,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:48,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:51,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:43:54,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:43:56,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:43:56,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:44:00,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:05,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:10,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:11,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:44:11,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:44:15,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:20,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:21,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:44:25,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:26,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:44:26,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:44:31,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:32,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:44:32,280 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:44:32,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:44:32,284 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:44:33,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:44:33,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:44:34,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:44:37,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:41,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:44:41,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:44:42,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:47,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:51,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:44:52,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:44:56,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:44:56,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:44:57,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:01,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:45:02,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:07,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:11,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:45:11,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:45:12,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:45:12,214 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:45:13,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:18,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:21,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:45:23,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:26,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:45:26,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:45:27,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:45:27,406 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:45:29,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:34,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:39,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:42,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:45:43,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:45:43,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:45:45,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:50,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:51,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:45:51,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:45:51,134 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:45:51,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:45:52,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:45:55,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:45:59,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:46:00,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:05,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:10,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:15,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:46:15,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:46:16,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:17,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:46:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:46:21,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:21,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:46:26,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:30,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:46:30,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:46:32,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:35,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:46:37,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:42,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:45,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:46:45,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:46:48,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:50,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:46:51,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:46:53,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:46:58,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:00,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:47:00,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:47:04,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:05,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:47:09,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:14,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:15,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:47:15,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:47:19,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:47:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:47:20,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:21,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:47:25,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:29,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:47:30,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:47:30,230 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:47:31,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:36,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:41,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:47:42,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:45,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:47:45,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:47:47,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:51,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:47:51,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:47:51,622 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:47:52,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:47:57,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:00,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:48:00,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:48:01,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:48:03,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:08,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:11,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:48:14,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:15,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:48:15,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:48:19,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:21,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:48:21,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:48:21,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:48:24,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:29,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:30,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:48:30,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:48:31,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:48:35,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:41,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:41,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:48:45,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:48:45,235 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:48:47,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:50,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:48:51,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:48:52,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:48:52,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:48:57,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:00,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:49:00,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:49:02,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:49:03,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:08,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:12,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:49:14,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:15,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:49:15,231 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:49:19,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:20,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:49:21,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:49:21,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:49:24,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:29,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:30,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:49:30,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:49:31,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:49:35,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:41,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:41,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:49:45,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:49:45,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:49:46,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:50,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:49:51,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:49:51,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:49:51,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:49:56,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:00,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:50:00,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:50:01,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:02,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:07,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:11,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:12,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:15,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:50:15,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:50:18,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:20,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:50:21,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:50:21,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:23,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:28,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:30,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:50:30,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:50:31,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:34,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:39,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:40,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:50:40,139 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:50:40,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:50:40,142 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:50:40,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:50:40,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:41,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:45,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:45,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:50:45,232 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:50:50,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:50:51,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:50:51,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:50:51,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:50:56,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:00,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:51:00,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:51:01,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:51:02,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:07,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:12,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:51:12,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:15,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:51:15,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:51:18,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:20,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:51:21,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:51:22,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:51:23,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:28,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:30,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:51:30,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:51:32,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:51:34,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:39,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:42,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:51:44,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:45,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:51:45,233 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:51:49,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:51:50,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:51:51,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:51:52,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:51:55,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:00,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:52:00,252 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:52:01,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:02,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:52:06,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:11,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:52:11,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:15,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:52:15,244 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:52:17,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:20,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:52:21,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:52:21,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:52:22,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:27,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:30,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:52:30,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:52:31,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:52:32,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:37,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:41,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:52:43,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:45,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:52:45,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:52:49,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:50,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:52:51,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:52:51,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:52:54,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:52:59,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:00,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:53:00,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:53:01,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:53:04,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:09,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:11,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:53:15,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:15,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:53:15,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:53:20,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:20,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:53:21,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:53:21,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:53:25,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:30,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:53:30,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:53:30,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:32,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:53:35,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:41,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:42,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:53:45,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:53:45,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:53:47,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:50,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:53:51,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:53:52,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:53:52,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:53:57,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:00,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:54:00,247 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:54:00,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:54:00,961 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:54:00,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:54:00,964 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:54:01,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:54:02,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:02,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:03,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:07,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:13,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:14,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:15,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:54:15,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:54:19,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:20,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:54:21,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:54:24,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:24,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:29,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:30,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:54:30,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:54:34,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:34,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:39,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:44,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:45,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:45,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:54:45,687 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:54:50,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:54:50,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:54:51,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:54:54,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:54:56,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:01,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:01,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:55:01,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:55:01,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:55:03,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:55:06,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:11,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:13,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:55:16,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:55:16,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:55:16,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:21,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:55:22,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:23,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:55:27,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:32,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:32,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:55:32,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:55:32,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:55:32,837 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:55:35,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:55:38,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:43,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:46,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:55:47,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:55:47,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:55:47,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:55:47,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:55:49,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:51,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:55:54,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:55:56,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:55:59,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:02,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:56:02,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:56:02,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:56:05,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:06,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:56:10,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:15,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:16,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:56:17,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:56:17,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:56:17,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:56:21,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:21,634 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:56:26,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:56:26,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:31,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:33,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:56:33,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:56:33,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:56:33,051 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:56:36,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:56:37,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:42,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:46,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:56:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:48,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:56:48,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:56:48,049 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:56:51,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:56:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:56:56,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:56:57,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:02,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:03,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:57:03,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:57:03,843 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:57:06,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:57:08,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:13,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:16,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:57:18,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:18,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:57:18,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:57:18,841 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:57:21,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:57:23,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:25,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 09:57:25,823 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 09:57:25,823 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 09:57:25,827 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 09:57:26,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 09:57:27,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:57:28,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:57:28,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:33,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:34,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:57:34,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:57:34,537 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:57:37,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:57:39,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:44,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:47,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:57:49,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:57:49,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:57:49,545 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:57:49,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:51,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:57:55,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:57:57,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:58:01,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:05,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:58:05,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:58:05,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:58:05,520 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:58:06,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:08,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:58:11,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:16,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:18,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:58:20,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:58:20,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:58:20,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:58:20,519 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:58:21,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:58:22,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:28,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:28,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:58:33,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:35,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:58:35,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:58:35,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:58:35,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:58:38,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:58:38,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:43,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:48,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:58:50,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:58:50,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:58:50,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:58:51,637 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:58:54,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:00,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:59:00,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:05,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:05,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:59:05,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:59:05,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:59:10,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:59:11,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:16,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:20,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:59:20,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:59:20,873 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:59:21,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:21,637 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:59:26,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:30,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:59:32,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:36,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:59:36,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:59:36,835 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:59:38,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:40,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:59:43,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:48,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:50,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 09:59:51,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 09:59:51,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 09:59:51,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 09:59:51,833 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 09:59:54,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 09:59:59,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:00,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:00:04,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:06,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:00:06,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:00:06,878 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:00:10,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:00:10,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:15,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:20,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:00:21,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:21,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:00:21,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:00:21,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:00:21,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:00:26,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:30,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:00:31,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:36,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:36,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:00:36,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:00:36,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:00:36,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:00:40,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:00:42,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:47,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:48,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:00:48,584 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:00:48,585 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:00:48,588 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:00:49,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:00:50,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:00:51,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:00:51,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:00:51,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:00:51,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:00:53,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:00:58,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:00,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:01:03,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:07,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:01:07,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:01:07,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:01:09,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:10,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:01:14,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:19,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:21,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:01:22,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:01:22,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:01:22,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:01:22,921 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:01:24,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:26,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:01:29,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:35,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:36,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:01:37,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:01:37,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:01:37,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:01:37,919 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:01:41,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:46,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:46,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:01:51,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:01:52,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:01:53,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:01:53,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:01:53,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:01:56,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:01:58,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:03,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:06,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:02:08,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:02:08,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:02:08,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:02:09,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:15,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:17,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:02:20,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:21,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:02:23,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:02:23,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:02:23,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:02:25,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:02:25,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:30,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:35,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:02:36,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:38,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:02:38,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:02:41,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:44,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:02:46,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:51,645 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:02:52,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:02:53,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:02:53,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:02:53,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:02:53,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:02:54,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:02:58,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:04,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:04,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:03:08,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:03:08,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:03:09,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:14,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:03:14,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:19,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:21,645 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:03:23,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:03:23,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:03:23,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:03:24,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:03:25,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:30,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:34,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:03:36,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:38,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:03:38,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:03:41,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:44,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:03:47,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:51,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:03:52,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:03:53,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:03:53,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:03:53,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:03:53,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:03:54,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:03:57,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:02,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:04,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:08,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:04:08,187 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:04:08,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:13,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:13,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:04:13,865 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:04:13,866 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:04:13,869 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:04:13,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:04:14,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:15,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:18,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:21,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:04:23,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:04:23,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:04:23,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:04:23,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:04:24,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:24,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:29,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:35,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:35,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:38,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:04:38,188 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:04:40,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:45,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:45,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:04:51,648 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:04:53,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:04:53,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:04:53,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:04:55,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:04:56,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:01,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:05,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:05:07,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:08,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:05:08,188 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:05:12,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:15,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:05:18,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:21,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:05:23,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:05:23,188 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:05:23,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:25,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:05:28,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:34,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:35,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:05:38,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:05:38,188 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:05:39,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:44,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:05:44,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:49,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:05:51,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:05:53,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:05:53,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:05:53,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:05:54,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:05:55,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:00,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:04,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:06:06,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:08,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:06:08,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:06:08,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:06:11,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:14,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:06:17,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:21,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:06:22,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:23,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:06:23,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:06:24,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:06:28,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:33,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:34,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:06:38,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:06:38,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:06:38,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:06:39,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:44,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:45,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:06:49,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:06:51,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:06:53,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:06:53,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:06:53,189 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:06:55,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:00,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:05,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:05,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:08,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:07:08,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:07:11,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:13,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:17,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:21,654 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:07:22,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:23,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:07:23,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:07:25,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:28,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:32,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:07:32,269 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:07:32,269 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:07:32,272 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:07:32,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:07:33,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:34,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:34,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:38,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:07:38,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:07:39,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:43,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:45,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:50,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:07:51,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:07:53,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:07:53,190 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:07:53,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:07:55,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:00,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:03,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:08:06,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:09,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:09,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:09,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:08:09,724 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:08:11,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:13,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:08:16,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:21,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:08:22,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:23,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:08:24,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:24,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:08:24,722 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:08:27,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:32,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:08:33,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:38,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:39,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:39,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:39,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:08:39,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:08:42,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:08:44,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:49,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:51,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:08:52,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:08:54,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:08:54,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:54,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:08:54,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:08:54,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:09:00,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:03,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:09:05,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:09,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:09:09,979 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:09:11,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:13,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:09:16,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:21,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:09:21,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:09:21,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:24,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:09:24,976 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:09:27,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:31,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:09:32,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:37,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:39,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:09:39,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:09:39,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:09:39,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:09:41,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:09:43,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:48,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:51,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:09:51,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:09:53,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:09:54,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:09:54,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:09:54,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:09:54,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:09:59,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:01,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:04,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:09,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:10:09,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:10:09,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:10:10,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:11,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:15,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:21,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:21,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:21,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:10:24,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:10:24,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:10:24,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:10:26,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:31,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:31,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:36,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:39,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:39,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:10:39,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:10:39,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:10:42,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:47,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:48,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:10:48,569 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:10:48,570 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:10:48,572 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:10:48,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:10:49,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:50,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:10:51,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:10:52,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:54,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:10:54,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:10:54,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:10:58,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:10:59,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:11:03,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:08,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:09,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:11:09,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:11:09,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:11:09,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:11:14,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:19,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:11:19,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:21,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:11:24,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:24,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:11:24,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:11:24,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:11:24,988 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:11:29,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:11:30,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:35,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:39,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:11:39,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:11:39,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:11:39,985 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:11:40,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:45,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:49,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:11:51,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:51,660 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:11:56,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:11:56,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:11:56,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:11:56,584 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:11:56,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:11:59,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:12:01,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:06,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:09,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:12:11,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:12:11,580 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:12:12,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:17,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:19,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:12:21,661 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:12:23,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:26,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:12:26,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:12:26,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:12:26,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:12:28,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:29,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:12:33,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:39,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:39,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:12:41,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:12:41,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:12:41,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:12:41,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:12:44,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:49,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:12:50,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:51,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:12:55,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:12:56,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:12:56,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:12:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:12:56,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:12:59,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:13:01,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:06,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:09,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:13:11,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:13:11,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:13:11,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:16,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:20,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:13:21,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:13:22,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:26,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:13:26,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:13:28,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:30,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:13:33,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:39,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:13:39,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:41,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:13:41,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:13:44,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:49,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:13:50,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:51,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:13:55,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:13:56,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:13:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:13:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:13:56,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:13:59,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:01,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:06,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:07,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:14:07,097 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:14:07,098 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:14:07,100 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:14:07,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:14:08,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:09,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:11,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:14:11,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:14:11,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:17,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:17,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:21,664 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:14:23,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:26,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:14:26,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:14:26,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:14:26,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:14:27,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:28,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:33,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:37,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:39,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:41,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:14:41,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:14:44,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:47,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:14:50,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:51,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:14:55,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:14:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:14:56,655 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:14:57,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:15:00,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:05,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:07,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:15:10,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:11,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:15:11,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:15:16,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:17,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:15:21,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:15:22,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:26,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:15:26,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:15:27,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:27,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:15:32,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:38,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:15:38,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:41,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:15:41,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:15:43,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:48,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:51,666 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:15:52,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:15:54,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:15:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:15:56,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:15:59,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:02,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:16:05,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:11,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:11,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:16:11,656 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:16:12,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:16:16,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:21,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:16:21,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:16:22,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:27,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:28,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:16:28,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:16:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:16:28,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:16:31,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:16:33,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:38,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:41,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:16:43,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:16:43,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:16:44,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:50,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:51,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:16:51,667 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:16:55,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:16:58,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:16:58,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:16:58,289 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:16:59,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:00,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:05,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:09,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:11,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:13,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:17:13,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:17:16,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:19,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:21,667 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:17:21,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:26,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:28,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:17:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:17:28,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:17:28,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:17:28,689 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:17:28,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:17:28,692 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:17:28,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:17:29,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:30,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:31,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:36,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:39,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:42,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:43,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:17:43,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:17:47,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:50,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:17:51,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:17:52,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:57,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:17:58,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:17:58,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:17:58,304 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:18:00,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:03,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:08,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:10,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:13,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:18:13,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:18:13,300 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:18:14,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:20,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:20,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:21,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:18:25,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:18:28,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:28,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:18:28,300 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:18:31,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:36,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:38,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:41,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:43,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:18:43,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:18:43,300 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:18:47,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:48,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:51,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:18:52,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:18:57,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:18:58,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:18:58,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:18:58,301 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:18:58,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:03,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:07,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:19:08,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:13,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:19:13,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:19:13,303 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:19:14,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:17,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:19:20,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:21,670 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:19:25,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:27,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:19:28,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:19:28,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:19:28,301 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:19:31,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:36,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:37,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:19:42,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:43,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:19:43,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:19:43,301 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:19:48,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:19:48,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:51,670 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:19:53,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:19:58,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:19:58,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:19:58,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:19:58,301 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:19:59,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:04,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:08,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:09,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:13,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:20:13,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:20:13,304 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:20:15,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:18,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:21,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:21,671 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:20:27,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:28,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:20:28,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:20:28,302 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:20:28,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:32,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:37,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:38,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:42,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:44,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:20:44,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:20:44,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:20:44,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:20:45,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:20:45,607 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:20:45,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:20:45,610 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:20:46,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:20:47,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:48,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:48,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:51,672 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:20:53,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:20:56,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:20:59,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:20:59,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:20:59,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:20:59,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:04,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:06,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:21:10,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:14,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:21:14,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:21:14,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:21:14,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:21:15,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:16,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:21:20,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:21,674 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:21:25,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:26,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:21:29,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:21:29,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:21:29,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:21:31,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:35,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:21:36,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:41,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:44,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:21:44,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:21:44,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:21:44,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:21:45,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:21:47,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:51,673 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:21:53,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:55,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:21:59,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:21:59,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:21:59,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:21:59,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:22:04,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:06,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:22:09,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:14,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:22:14,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:22:14,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:22:14,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:22:15,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:16,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:22:20,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:21,675 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:22:25,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:26,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:22:29,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:22:29,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:22:29,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:22:30,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:35,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:36,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:22:40,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:44,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:22:44,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:22:44,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:22:46,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:22:46,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:51,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:51,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:22:56,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:22:57,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:22:59,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:22:59,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:22:59,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:23:02,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:06,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:23:08,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:13,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:14,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:23:14,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:23:14,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:23:16,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:23:18,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:21,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:23:23,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:26,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:23:28,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:29,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:23:29,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:23:29,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:23:34,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:34,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:23:39,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:44,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:23:44,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:23:44,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:23:44,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:44,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:23:49,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:51,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:23:54,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:23:55,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:23:59,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:23:59,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:23:59,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:23:59,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:24:00,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:03,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:24:03,431 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:24:03,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:24:03,434 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:24:03,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:24:04,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:05,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:06,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:11,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:13,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:14,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:24:14,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:24:14,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:24:14,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:24:16,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:21,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:24:22,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:24,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:27,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:31,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:24:31,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:24:33,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:34,105 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:38,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:43,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:46,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:24:46,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:24:48,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:48,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:51,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:24:53,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:24:58,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:24:58,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:01,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:25:01,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:25:04,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:08,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:25:09,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:14,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:16,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:25:16,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:25:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:25:16,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:25:18,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:25:20,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:21,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:25:25,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:28,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:25:31,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:31,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:25:31,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:25:36,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:38,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:25:41,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:46,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:25:46,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:25:46,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:25:47,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:48,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:25:51,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:25:52,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:58,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:25:58,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:26:01,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:26:01,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:26:03,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:06,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:26:08,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:13,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:16,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:26:16,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:26:16,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:26:19,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:21,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:26:24,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:26,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:26:29,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:31,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:26:31,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:26:35,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:38,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:26:40,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:45,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:46,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:26:46,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:26:51,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:26:51,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:26:56,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:01,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:27:01,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:27:01,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:27:02,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:04,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:27:07,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:12,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:14,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:27:16,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:27:16,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:27:17,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:21,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:27:23,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:24,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:27:28,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:31,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:27:31,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:27:31,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:27:33,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:34,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:27:38,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:43,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:46,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:27:46,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:27:48,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:51,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:27:54,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:27:59,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:01,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:28:01,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:28:04,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:09,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:14,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:16,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:28:16,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:28:20,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:21,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:28:25,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:25,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:28:25,964 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:28:25,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:28:25,968 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:28:26,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:28:27,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:28:28,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:28:30,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:31,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:28:31,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:28:36,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:37,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:28:41,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:46,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:46,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:28:46,582 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:28:47,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:28:51,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:28:51,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:57,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:28:58,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:29:01,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:29:01,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:29:02,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:06,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:29:08,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:13,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:16,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:29:16,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:29:16,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:29:18,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:21,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:29:24,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:26,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:29:30,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:31,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:29:31,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:29:35,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:36,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:29:40,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:45,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:46,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:29:46,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:29:46,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:29:51,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:29:51,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:29:56,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:01,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:30:01,579 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:30:01,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:06,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:08,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:30:12,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:16,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:30:16,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:30:16,582 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:30:16,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:30:17,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:18,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:30:21,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:30:23,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:28,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:28,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:30:31,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:30:31,582 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:30:33,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:39,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:30:39,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:44,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:47,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:30:47,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:30:47,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:30:49,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:51,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:30:51,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:30:54,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:30:59,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:01,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:02,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:31:02,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:31:02,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:31:05,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:10,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:11,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:15,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:17,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:31:17,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:31:17,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:31:19,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:21,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:21,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:31:26,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:32,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:31:32,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:31:32,195 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:31:32,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:32,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:37,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:40,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:43,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:48,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:48,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:31:48,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:31:48,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:31:50,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:51,689 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:31:53,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:31:56,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:31:58,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:03,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:32:03,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:32:03,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:32:04,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:05,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:32:05,162 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:32:05,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:32:05,165 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:32:05,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:32:06,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:07,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:10,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:15,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:16,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:18,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:32:18,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:32:18,728 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:32:20,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:21,689 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:32:26,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:26,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:31,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:33,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:32:33,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:32:33,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:32:33,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:32:36,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:37,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:42,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:46,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:47,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:48,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:32:48,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:32:48,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:32:51,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:32:53,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:32:57,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:32:59,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:03,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:33:03,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:33:03,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:33:03,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:33:05,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:33:05,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:10,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:15,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:33:15,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:18,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:33:18,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:33:18,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:33:21,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:21,691 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:33:26,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:27,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:33:31,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:33,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:33:33,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:33:33,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:33:33,994 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:33:35,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:33:37,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:42,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:45,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:33:47,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:48,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:33:48,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:33:48,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:33:51,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:33:52,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:33:55,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:33:58,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:03,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:03,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:34:03,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:34:03,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:34:05,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:34:09,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:14,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:34:15,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:18,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:34:18,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:34:18,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:34:20,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:21,693 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:34:24,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:34:25,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:30,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:32,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:34:33,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:34:33,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:34:33,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:34:36,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:41,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:42,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:34:46,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:50,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:34:50,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:34:50,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:34:50,512 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:34:51,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:34:52,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:34:52,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:34:57,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:03,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:03,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:05,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:35:05,511 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:35:08,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:13,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:15,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:18,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:21,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:35:21,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:35:21,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:35:21,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:35:21,695 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:35:22,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:35:22,615 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:35:22,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:35:22,619 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:35:23,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:35:23,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:24,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:24,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:29,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:33,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:35,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:36,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:35:36,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:35:36,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:35:40,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:43,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:45,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:50,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:35:51,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:35:51,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:35:51,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:35:51,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:35:51,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:35:53,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:35:56,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:02,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:03,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:36:06,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:36:06,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:36:06,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:36:07,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:13,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:13,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:36:18,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:21,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:36:21,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:36:21,328 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:36:21,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:36:23,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:36:23,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:28,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:33,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:36:34,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:36,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:36:36,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:36:36,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:36:39,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:43,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:36:45,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:50,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:36:51,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:36:51,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:36:51,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:36:51,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:36:53,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:36:55,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:00,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:02,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:37:06,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:37:06,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:37:06,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:37:06,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:11,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:16,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:21,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:37:21,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:37:21,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:37:21,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:21,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:37:23,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:37:26,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:32,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:33,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:37:36,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:37:36,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:37:36,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:37:37,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:41,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:37:43,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:48,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:51,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:37:51,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:37:51,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:37:51,329 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:37:51,698 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:37:53,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:37:58,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:01,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:04,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:06,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:38:06,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:38:06,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:38:09,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:11,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:20,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:21,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:38:21,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:38:21,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:38:21,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:21,698 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:38:25,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:29,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:30,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:35,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:37,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:38:37,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:38:37,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:38:37,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:38:39,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:41,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:46,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:48,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:38:48,720 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:38:48,721 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:38:48,725 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:38:48,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:38:49,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:50,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:38:51,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:38:51,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:52,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:38:52,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:38:52,379 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:38:57,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:38:59,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:39:03,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:07,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:39:07,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:39:07,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:39:07,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:39:09,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:09,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:39:14,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:19,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:19,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:39:21,699 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:39:22,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:39:22,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:39:22,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:39:24,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:30,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:39:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:35,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:37,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:39:37,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:39:37,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:39:40,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:39:40,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:45,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:49,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:39:51,700 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:39:51,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:52,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:39:52,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:39:52,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:39:57,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:39:59,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:40:02,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:07,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:40:07,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:40:07,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:40:08,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:09,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:40:13,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:18,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:19,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:40:21,700 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:40:22,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:40:22,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:40:22,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:40:24,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:29,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:40:29,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:34,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:37,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:40:37,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:40:37,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:40:39,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:40:40,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:45,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:49,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:40:50,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:51,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:40:52,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:40:52,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:40:52,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:40:56,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:40:59,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:41:02,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:07,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:07,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:41:07,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:41:07,383 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:41:09,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:41:13,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:18,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:19,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:41:21,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:41:22,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:41:22,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:41:22,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:41:24,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:29,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:41:30,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:35,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:37,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:41:37,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:41:37,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:41:39,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:41:40,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:45,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:47,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:41:50,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:41:51,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:41:52,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:41:52,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:41:52,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:41:56,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:00,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:42:02,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:07,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:42:07,195 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:42:07,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:42:07,198 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:42:07,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:42:07,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:42:07,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:42:07,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:08,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:42:08,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:42:12,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:18,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:18,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:42:21,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:42:22,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:42:22,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:42:22,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:42:23,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:28,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:42:28,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:33,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:37,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:42:37,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:42:37,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:42:37,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:42:39,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:44,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:47,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:42:49,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:51,703 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:42:52,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:42:52,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:42:52,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:42:55,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:42:57,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:00,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:05,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:07,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:43:07,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:43:07,383 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:43:07,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:11,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:16,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:17,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:21,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:43:22,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:43:22,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:43:22,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:43:22,387 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:43:22,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:27,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:28,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:33,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:35,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:37,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:43:37,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:43:37,383 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:43:38,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:43,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:45,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:49,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:51,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:43:53,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:43:53,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:43:53,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:43:53,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:43:54,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:43:55,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:43:59,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:05,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:06,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:44:08,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:44:08,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:44:08,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:44:10,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:16,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:16,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:44:21,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:21,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:44:23,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:44:23,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:44:23,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:44:26,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:44:26,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:31,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:34,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:44:37,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:38,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:44:38,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:44:38,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:44:42,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:44,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:44:48,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:51,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:44:53,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:44:53,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:44:53,429 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:44:53,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:44:54,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:44:58,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:03,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:04,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:08,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:45:08,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:45:08,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:45:09,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:14,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:15,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:20,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:21,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:45:21,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:45:21,789 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:45:21,790 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:45:21,792 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:45:22,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:45:22,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:23,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:45:23,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:45:23,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:45:23,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:25,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:30,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:33,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:36,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:38,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:45:38,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:45:38,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:45:41,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:43,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:47,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:51,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:45:53,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:45:53,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:45:53,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:45:53,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:45:53,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:45:58,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:03,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:03,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:46:08,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:46:08,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:46:08,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:46:09,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:13,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:46:15,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:20,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:21,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:46:23,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:46:23,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:46:23,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:46:24,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:46:25,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:30,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:34,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:46:35,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:38,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:46:38,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:46:38,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:46:41,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:46,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:50,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:46:51,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:46:51,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:46:53,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:46:53,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:46:53,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:46:57,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:00,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:47:02,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:08,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:47:08,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:47:08,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:47:08,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:10,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:47:13,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:19,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:20,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:47:21,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:47:23,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:47:23,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:47:23,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:47:24,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:30,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:47:34,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:38,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:47:38,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:47:38,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:47:40,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:47:40,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:45,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:50,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:47:51,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:51,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:47:53,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:47:53,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:47:53,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:47:56,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:47:58,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:02,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:07,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:08,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:08,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:48:08,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:48:08,433 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:48:13,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:18,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:19,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:21,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:48:23,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:48:23,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:48:23,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:48:24,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:30,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:30,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:35,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:38,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:38,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:48:38,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:48:38,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:48:40,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:45,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:46,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:48:46,793 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:48:46,793 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:48:46,796 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:48:47,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:48:48,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:49,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:48:50,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:51,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:48:53,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:48:53,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:48:53,436 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:48:56,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:48:58,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:49:01,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:07,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:08,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:49:08,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:08,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:49:08,433 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:49:12,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:18,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:18,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:49:21,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:49:23,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:24,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:49:24,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:49:28,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:49:28,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:33,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:38,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:49:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:39,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:49:39,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:49:39,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:44,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:46,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:49:50,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:51,716 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:49:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:49:54,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:49:54,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:49:55,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:49:56,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:00,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:06,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:06,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:09,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:50:09,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:50:09,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:50:11,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:16,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:16,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:21,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:50:22,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:50:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:50:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:50:24,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:50:26,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:27,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:32,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:35,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:38,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:39,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:50:39,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:50:39,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:50:43,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:46,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:49,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:51,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:50:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:50:54,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:50:54,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:50:54,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:50:56,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:50:59,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:04,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:06,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:51:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:51:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:51:09,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:51:10,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:16,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:16,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:51:21,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:21,721 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:51:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:51:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:51:24,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:51:26,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:51:26,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:31,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:36,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:51:37,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:39,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:51:39,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:51:39,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:51:42,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:46,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:51:48,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:51,722 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:51:53,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:51:54,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:51:54,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:51:54,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:51:56,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:51:58,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:03,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:04,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:52:04,062 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:52:04,062 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:52:04,065 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:52:04,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:52:05,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:06,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:09,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:09,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:52:09,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:52:09,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:52:14,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:16,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:19,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:21,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:52:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:52:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:52:24,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:52:24,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:26,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:29,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:35,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:36,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:39,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:52:39,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:52:39,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:52:40,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:44,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:46,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:51,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:52:51,724 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:52:54,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:52:54,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:52:54,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:52:54,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:52:56,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:01,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:05,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:53:06,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:09,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:53:09,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:53:09,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:53:12,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:15,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:53:18,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:21,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:53:23,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:24,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:53:24,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:53:24,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:53:24,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:53:28,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:34,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:53:34,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:39,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:41,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:53:41,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:53:41,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:53:44,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:53:45,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:50,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:53:51,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:53:54,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:53:56,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:53:56,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:53:56,161 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:53:56,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:01,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:04,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:54:07,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:11,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:54:11,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:54:11,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:54:11,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:54:12,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:14,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:54:17,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:21,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:54:23,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:24,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:54:26,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:54:26,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:54:26,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:54:28,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:32,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:54:33,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:38,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:41,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:54:41,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:54:41,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:54:41,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:54:42,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:54:43,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:48,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:51,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:54:52,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:54:53,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:54:56,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:54:56,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:54:56,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:54:59,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:02,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:55:04,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:09,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:11,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:55:11,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:55:11,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:55:12,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:55:15,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:20,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:20,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:55:20,878 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:55:20,879 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:55:20,881 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:55:20,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:55:21,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:55:23,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:55:25,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:26,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:55:26,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:55:26,162 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:55:30,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:33,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:55:35,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:41,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:55:41,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:55:41,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:55:41,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:43,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:55:46,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:51,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:55:51,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:55:53,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:55:56,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:55:56,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:55:56,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:55:57,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:03,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:03,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:56:08,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:11,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:56:11,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:56:11,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:56:12,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:56:13,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:18,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:21,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:56:22,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:56:23,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:26,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:56:26,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:56:26,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:56:29,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:32,500 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:56:35,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:40,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:56:40,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:41,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:56:41,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:56:41,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:56:46,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:50,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:56:51,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:56:51,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:56:56,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:56:56,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:56:56,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:56:57,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:00,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:57:02,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:07,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:10,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:57:11,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:57:11,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:57:11,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:57:13,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:18,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:20,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:57:21,732 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:57:23,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:26,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:57:26,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:57:26,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:57:29,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:30,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:57:34,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:39,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:41,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:57:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:57:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:57:41,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:57:45,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:51,730 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:57:53,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:57:55,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:57:56,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:57:56,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:58:01,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:03,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:58:06,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:11,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:58:11,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:58:11,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:13,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:58:16,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:21,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:58:21,731 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:58:21,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:26,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:58:26,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:58:27,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:31,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:58:33,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:38,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:39,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 10:58:39,777 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 10:58:39,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 10:58:39,781 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 10:58:40,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 10:58:41,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:58:41,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:58:41,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:58:41,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:58:43,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:48,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:51,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:58:51,731 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:58:53,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:58:56,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:58:56,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:58:59,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:00,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:59:05,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:10,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:10,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:59:11,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:59:11,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:59:16,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:20,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:59:21,732 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:59:21,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:26,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:27,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:59:27,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:59:27,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:59:27,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:59:30,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:59:31,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:36,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:40,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:59:42,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:59:42,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:59:42,586 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:59:42,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:47,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:50,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 10:59:51,732 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 10:59:53,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 10:59:57,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:59:57,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 10:59:57,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 10:59:57,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 10:59:59,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:01,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:00:04,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:09,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:00:10,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:12,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:00:12,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:12,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:00:15,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:19,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:00:20,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:21,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:00:26,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:27,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:00:27,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:27,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:00:27,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:29,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:00:32,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:37,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:41,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:00:42,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:42,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:00:42,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:42,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:00:42,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:47,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:51,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:00:51,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:00:53,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:00:57,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:57,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:00:57,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:00:57,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:00:58,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:01,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:01:04,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:10,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:11,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:01:12,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:01:12,587 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:01:15,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:20,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:21,736 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:01:25,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:01:26,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:27,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:01:27,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:01:31,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:34,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:01:36,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:41,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:42,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:01:42,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:01:42,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:01:42,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:01:44,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:01:46,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:51,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:01:52,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:01:54,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:01:57,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:01:57,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:01:57,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:01:57,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:01:58,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:03,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:03,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:02:03,815 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:02:03,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:02:03,818 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:02:03,962 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:02:04,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:05,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:08,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:12,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:02:12,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:02:12,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:02:12,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:02:14,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:15,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:19,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:21,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:02:25,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:25,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:27,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:02:27,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:02:27,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:02:30,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:35,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:36,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:41,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:42,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:02:42,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:02:42,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:02:45,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:46,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:51,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:02:52,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:02:55,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:02:57,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:02:57,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:02:57,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:02:57,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:02,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:05,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:03:08,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:12,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:03:12,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:03:12,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:03:12,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:03:13,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:15,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:03:18,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:21,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:03:23,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:25,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:03:27,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:03:27,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:03:27,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:03:28,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:34,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:35,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:03:39,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:42,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:03:42,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:03:42,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:03:43,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:03:44,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:49,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:51,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:03:53,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:03:55,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:03:57,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:03:57,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:03:57,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:04:00,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:05,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:04:06,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:11,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:12,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:04:12,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:04:12,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:04:12,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:04:16,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:21,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:04:22,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:23,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:04:27,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:04:27,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:04:27,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:04:28,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:33,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:04:33,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:38,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:42,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:04:42,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:04:42,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:04:43,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:04:44,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:49,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:51,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:04:53,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:04:54,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:04:57,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:04:57,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:04:57,779 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:05:00,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:03,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:05,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:10,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:12,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:05:12,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:05:12,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:05:13,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:16,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:20,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:05:20,538 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:05:20,539 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:05:20,541 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:05:21,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:05:21,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:21,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:21,753 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:05:22,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:26,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:28,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:05:28,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:05:28,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:05:28,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:05:31,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:32,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:37,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:41,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:42,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:43,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:05:43,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:05:43,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:05:48,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:51,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:05:51,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:05:53,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:58,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:05:58,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:05:58,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:05:58,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:05:58,969 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:06:01,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:06:04,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:09,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:11,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:06:13,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:06:13,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:06:13,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:06:13,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:06:14,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:19,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:21,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:06:21,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:06:24,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:28,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:06:28,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:06:28,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:06:28,969 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:06:30,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:31,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:06:35,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:40,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:06:40,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:43,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:06:43,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:06:43,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:06:46,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:50,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:06:51,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:51,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:06:56,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:06:58,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:06:58,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:06:58,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:07:00,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:02,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:07,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:09,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:13,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:13,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:07:13,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:07:13,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:07:18,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:19,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:21,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:07:23,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:28,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:07:28,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:07:28,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:07:29,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:29,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:34,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:39,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:39,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:43,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:07:43,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:07:43,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:07:43,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:07:45,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:49,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:51,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:07:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:07:57,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:07:58,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:07:58,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:07:58,969 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:08:02,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:07,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:08,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:13,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:14,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:08:14,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:08:14,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:08:14,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:08:17,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:18,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:21,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:08:24,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:27,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:29,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:08:29,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:08:29,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:08:29,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:34,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:35,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:08:35,827 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:08:35,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:08:35,830 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:08:36,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:08:37,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:38,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:40,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:44,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:08:44,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:08:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:08:44,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:08:46,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:47,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:51,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:51,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:08:57,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:08:57,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:08:59,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:08:59,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:08:59,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:09:02,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:08,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:09:08,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:13,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:14,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:09:14,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:09:14,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:09:14,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:09:16,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:09:18,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:21,748 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:09:24,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:28,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:09:29,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:09:29,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:09:30,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:35,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:38,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:09:40,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:44,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:09:44,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:09:44,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:09:44,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:09:46,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:47,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:09:51,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:09:51,748 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:09:56,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:01,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:10:01,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:10:01,009 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:10:02,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:07,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:12,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:16,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:10:16,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:10:16,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:10:17,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:19,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:10:21,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:10:23,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:28,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:29,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:10:31,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:10:31,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:10:31,593 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:10:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:38,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:41,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:10:44,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:46,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:10:46,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:10:46,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:10:50,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:10:51,750 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:10:51,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:10:55,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:01,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:01,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:11:01,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:11:01,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:11:01,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:11:06,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:12,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:11:12,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:16,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:11:16,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:11:16,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:11:17,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:21,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:11:22,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:11:23,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:28,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:31,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:11:31,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:11:31,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:11:32,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:11:33,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:38,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:42,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:11:44,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:46,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:11:46,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:11:46,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:11:49,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:11:51,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:11:52,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:11:55,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:00,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:01,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:12:01,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:12:01,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:12:02,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:06,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:12,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:12,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:16,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:12:16,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:12:16,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:12:17,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:19,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:12:19,380 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:12:19,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:12:19,383 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:12:19,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:12:20,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:21,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:12:22,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:23,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:28,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:30,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:31,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:12:31,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:12:31,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:12:34,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:39,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:40,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:45,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:47,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:12:47,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:12:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:12:47,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:12:49,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:12:50,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:51,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:12:55,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:12:59,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:01,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:02,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:13:02,484 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:13:06,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:09,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:11,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:16,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:17,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:13:17,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:13:17,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:13:19,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:21,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:13:22,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:28,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:30,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:32,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:13:32,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:13:33,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:38,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:40,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:43,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:47,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:13:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:13:47,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:13:48,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:50,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:51,764 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:13:53,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:13:58,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:13:59,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:02,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:14:02,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:14:04,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:08,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:14:10,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:15,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:17,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:14:17,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:14:17,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:14:20,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:14:20,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:21,753 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:14:25,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:28,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:14:31,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:32,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:14:32,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:14:36,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:38,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:14:42,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:14:47,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:14:47,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:14:47,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:48,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:14:51,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:14:52,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:57,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:14:58,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:02,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:15:02,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:15:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:15:03,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:08,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:08,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:13,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:17,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:15:17,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:15:17,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:15:17,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:19,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:21,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:15:25,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:27,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:31,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:32,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:15:32,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:15:32,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:15:36,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:37,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:15:37,046 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:15:37,047 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:15:37,048 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:15:37,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:15:37,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:39,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:42,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:47,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:15:47,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:15:47,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:15:48,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:15:51,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:15:52,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:57,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:15:58,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:16:02,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:16:02,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:16:02,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:16:03,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:09,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:10,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:16:14,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:17,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:16:17,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:16:17,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:16:19,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:21,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:16:24,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:16:25,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:30,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:32,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:16:32,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:16:32,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:16:34,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:16:35,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:40,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:44,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:16:46,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:47,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:16:47,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:16:47,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:16:51,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:16:51,758 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:16:54,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:16:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:02,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:17:02,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:17:02,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:17:02,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:17:02,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:07,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:12,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:17:13,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:17,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:17:17,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:17:17,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:17:18,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:21,758 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:17:22,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:17:23,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:28,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:32,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:17:32,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:17:32,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:17:32,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:17:34,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:39,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:42,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:17:45,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:47,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:17:47,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:17:47,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:17:50,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:17:51,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:17:52,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:17:55,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:01,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:02,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:18:02,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:18:02,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:18:02,473 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:18:06,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:12,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:12,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:18:17,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:17,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:18:17,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:18:17,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:18:21,760 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:18:22,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:18:22,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:27,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:32,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:18:32,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:18:32,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:18:32,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:18:33,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:38,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:42,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:18:44,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:47,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:18:47,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:18:47,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:18:49,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:51,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:18:52,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:18:54,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:18:59,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:00,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:19:00,560 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:19:00,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:19:00,564 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:19:01,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:19:02,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:02,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:19:02,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:19:02,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:19:03,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:05,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:11,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:12,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:16,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:17,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:19:17,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:19:17,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:19:21,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:19:21,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:22,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:26,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:31,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:32,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:19:32,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:19:32,474 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:19:32,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:37,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:42,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:42,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:47,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:19:47,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:19:47,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:19:47,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:51,762 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:19:52,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:19:52,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:19:57,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:02,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:02,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:20:02,476 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:20:02,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:20:03,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:08,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:12,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:20:14,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:19,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:19,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:19,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:20:19,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:20:20,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:21,762 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:20:23,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:20:25,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:31,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:33,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:20:34,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:34,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:34,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:20:34,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:20:36,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:40,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:20:41,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:46,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:49,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:49,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:20:49,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:20:49,059 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:20:50,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:20:51,763 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:20:52,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:20:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:00,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:21:02,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:04,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:21:04,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:21:04,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:21:08,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:10,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:21:13,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:18,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:19,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:21:19,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:21:19,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:21:20,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:21:21,764 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:21:23,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:29,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:30,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:21:34,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:21:34,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:21:34,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:21:35,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:40,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:21:41,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:46,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:48,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:21:49,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:21:49,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:21:49,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:21:51,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:51,764 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:21:56,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:21:58,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:22:01,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:04,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:22:04,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:22:04,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:22:07,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:08,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:22:12,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:16,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:22:16,961 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:22:16,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:22:16,964 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:22:17,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:22:17,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:18,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:22:19,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:22:19,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:22:19,057 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:22:21,765 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:22:23,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:28,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:22:29,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:34,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:35,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:22:35,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:22:35,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:22:35,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:22:39,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:22:40,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:45,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:49,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:22:50,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:22:50,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:22:50,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:22:50,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:22:51,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:51,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:22:56,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:22:59,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:23:01,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:05,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:23:05,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:23:05,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:23:05,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:23:07,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:09,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:23:12,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:17,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:19,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:23:20,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:23:20,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:23:20,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:23:20,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:23:21,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:23:22,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:26,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:23:28,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:33,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:35,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:23:35,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:23:35,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:23:35,398 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:23:38,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:23:38,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:43,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:48,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:23:49,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:50,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:23:50,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:23:51,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:23:54,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:23:58,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:24:00,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:05,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:05,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:24:05,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:24:06,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:24:10,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:15,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:18,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:24:20,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:24:20,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:24:21,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:21,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:24:26,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:30,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:24:32,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:35,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:24:35,396 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:24:37,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:41,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:24:42,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:47,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:50,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:24:50,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:24:51,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:24:53,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:24:53,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:24:58,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:03,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:25:04,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:05,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:25:05,397 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:25:09,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:13,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:25:15,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:20,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:20,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:25:20,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:25:20,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:25:20,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:25:21,769 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:25:23,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:25:26,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:31,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:35,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:25:35,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:25:35,738 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:25:36,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:41,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:43,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:25:43,184 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:25:43,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:25:43,187 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:25:43,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:25:45,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:25:47,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:50,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:25:50,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:25:50,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:25:50,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:25:51,770 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:25:52,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:25:55,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:25:57,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:02,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:05,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:26:05,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:26:05,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:26:08,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:13,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:16,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:26:18,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:20,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:26:20,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:26:21,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:26:23,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:26,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:26:28,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:33,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:35,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:26:35,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:26:36,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:26:38,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:43,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:46,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:26:49,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:26:50,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:26:51,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:26:54,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:26:58,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:27:00,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:05,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:27:05,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:27:05,743 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:27:06,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:09,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:27:11,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:16,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:19,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:27:20,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:27:20,740 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:27:21,772 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:27:22,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:27,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:29,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:27:32,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:36,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:27:36,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:27:36,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:27:36,355 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:27:38,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:39,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:27:43,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:49,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:27:49,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:51,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:27:51,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:27:51,354 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:27:51,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:27:54,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:27:59,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:28:00,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:05,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:07,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:28:07,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:28:07,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:28:07,171 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:28:10,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:11,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:28:15,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:21,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:21,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:28:21,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:28:22,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:28:22,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:28:26,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:31,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:31,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:28:36,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:38,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:28:38,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:28:38,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:28:38,402 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:28:41,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:28:42,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:47,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:50,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:28:51,774 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:28:52,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:28:53,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:28:53,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:28:53,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:28:53,403 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:28:58,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:00,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:04,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:09,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:09,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:09,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:29:09,148 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:29:10,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:10,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:29:10,432 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:29:10,433 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:29:10,437 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:29:10,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:29:10,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:11,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:15,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:20,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:21,774 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:29:23,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:24,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:24,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:24,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:29:24,147 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:29:25,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:30,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:33,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:35,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:40,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:40,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:40,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:29:40,262 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:29:41,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:43,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:46,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:51,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:29:51,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:29:53,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:29:55,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:29:55,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:29:55,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:29:57,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:02,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:03,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:30:07,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:10,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:30:10,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:30:10,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:30:10,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:30:12,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:13,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:30:17,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:21,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:30:23,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:30:23,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:25,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:30:25,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:30:25,262 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:30:29,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:33,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:30:35,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:40,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:30:40,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:30:40,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:30:40,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:43,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:30:46,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:51,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:30:51,776 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:30:53,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:30:55,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:30:55,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:30:55,263 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:30:56,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:01,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:03,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:31:06,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:10,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:31:10,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:31:10,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:31:12,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:13,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:31:17,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:21,777 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:31:22,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:25,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:31:25,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:31:25,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:31:25,592 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:31:28,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:29,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:31:33,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:38,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:39,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:31:40,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:31:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:31:40,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:31:40,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:31:43,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:48,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:49,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:31:51,777 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:31:54,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:31:55,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:31:55,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:31:55,746 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:31:59,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:31:59,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:04,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:07,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:10,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:32:10,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:32:10,746 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:32:10,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:15,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:17,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:21,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:32:21,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:25,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:32:25,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:32:25,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:32:25,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:32:26,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:27,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:31,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:36,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:32:36,506 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:32:36,507 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:32:36,510 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:32:37,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:32:37,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:37,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:38,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:40,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:32:40,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:32:40,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:32:42,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:47,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:48,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:51,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:32:53,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:32:55,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:32:55,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:32:55,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:32:55,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:32:57,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:32:59,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:04,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:07,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:33:09,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:10,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:33:10,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:33:10,777 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:33:15,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:20,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:21,779 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:33:25,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:25,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:33:25,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:33:25,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:33:27,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:33:30,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:36,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:37,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:33:40,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:33:40,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:33:40,778 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:33:41,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:46,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:48,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:33:51,779 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:33:52,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:33:56,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:33:56,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:33:56,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:33:56,529 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:33:57,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:00,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:34:02,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:07,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:09,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:34:11,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:34:11,526 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:34:13,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:19,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:34:19,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:21,780 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:34:24,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:27,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:34:27,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:34:27,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:34:27,361 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:34:29,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:34:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:34,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:40,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:41,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:34:42,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:34:42,359 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:34:45,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:51,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:51,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:34:51,780 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:34:56,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:34:57,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:34:57,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:34:57,731 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:34:59,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:35:01,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:06,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:09,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:35:12,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:12,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:35:12,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:35:12,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:35:17,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:19,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:35:21,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:35:23,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:27,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:35:27,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:35:27,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:35:27,732 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:35:29,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:29,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:35:34,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:39,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:35:40,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:42,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:35:42,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:35:42,725 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:35:45,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:49,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:35:51,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:51,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:35:56,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:35:57,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:35:57,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:35:57,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:35:57,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:36:02,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:36:02,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:07,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:09,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:36:09,291 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:36:09,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:36:09,294 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:36:10,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:36:10,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:36:12,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:12,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:36:12,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:36:13,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:18,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:20,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:36:21,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:36:23,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:27,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:27,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:27,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:36:27,956 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:36:29,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:30,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:36:34,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:39,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:36:40,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:42,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:42,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:42,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:36:42,956 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:36:45,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:49,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:36:51,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:51,783 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:36:56,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:36:57,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:57,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:36:57,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:36:57,957 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:36:59,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:02,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:07,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:09,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:12,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:37:12,955 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:37:13,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:18,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:19,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:21,784 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:37:24,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:27,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:37:27,955 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:37:27,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:37:29,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:35,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:35,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:36,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:40,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:41,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:42,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:42,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:37:42,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:37:42,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:37:42,993 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:37:46,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:51,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:51,784 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:37:51,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:37:56,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:37:59,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:37:59,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:37:59,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:37:59,135 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:38:01,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:38:02,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:07,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:12,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:38:12,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:14,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:38:14,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:38:14,133 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:38:17,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:21,788 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:38:22,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:38:22,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:27,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:29,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:38:29,135 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:38:32,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:38:33,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:38,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:42,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:38:43,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:44,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:38:44,133 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:38:48,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:51,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:38:52,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:38:53,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:58,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:38:59,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:38:59,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:38:59,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:38:59,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:39:02,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:04,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:09,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:12,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:14,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:39:14,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:39:14,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:39:14,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:39:14,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:19,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:21,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:39:21,593 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:39:21,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:39:21,597 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:39:21,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:39:22,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:39:22,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:24,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:24,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:29,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:30,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:39:30,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:39:30,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:39:33,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:35,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:40,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:41,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:45,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:39:45,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:39:45,339 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:39:46,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:51,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:39:51,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:39:51,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:39:56,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:00,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:40:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:40:00,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:40:01,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:40:01,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:06,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:11,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:40:11,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:15,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:40:15,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:40:15,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:40:17,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:21,787 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:40:22,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:40:22,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:27,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:30,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:40:30,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:40:30,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:40:32,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:40:33,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:42,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:40:43,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:45,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:40:45,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:40:45,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:40:49,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:51,787 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:40:52,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:40:54,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:40:59,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:41:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:41:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:41:00,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:41:02,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:41:05,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:11,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:12,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:41:15,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:41:15,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:41:15,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:41:16,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:21,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:21,788 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:41:22,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:41:26,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:30,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:41:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:41:30,340 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:41:32,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:37,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:42,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:41:45,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:41:45,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:41:48,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:51,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:41:52,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:41:53,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:41:58,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:42:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:42:00,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:42:02,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:42:04,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:09,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:11,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:42:15,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:42:15,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:42:15,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:42:15,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:21,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:21,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:42:21,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:42:26,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:42:30,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:42:30,341 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:42:31,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:42:32,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:37,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:42,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:42:43,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:42:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:42:45,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:42:48,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:51,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:42:52,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:42:53,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:42:58,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:43:00,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:43:00,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:43:00,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:43:00,705 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:43:00,706 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:43:00,709 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:43:01,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:43:02,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:03,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:04,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:09,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:12,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:43:15,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:43:15,342 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:43:15,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:20,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:21,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:43:24,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:30,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:43:30,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:43:30,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:43:31,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:34,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:36,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:42,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:42,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:45,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:43:45,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:43:45,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:43:47,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:51,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:43:52,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:43:52,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:43:57,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:00,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:44:00,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:44:00,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:44:00,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:44:02,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:44:03,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:08,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:12,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:44:14,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:15,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:44:15,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:44:15,452 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:44:19,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:21,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:44:22,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:44:24,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:29,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:30,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:44:30,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:44:30,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:44:30,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:44:32,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:44:35,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:40,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:42,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:44:45,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:44:45,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:44:45,453 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:44:45,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:50,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:44:51,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:44:54,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:44:55,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:00,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:45:00,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:45:00,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:45:00,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:45:01,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:04,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:45:06,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:11,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:14,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:45:15,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:45:15,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:45:15,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:45:16,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:21,793 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:45:21,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:22,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:45:26,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:30,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:45:30,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:45:30,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:45:32,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:45:32,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:37,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:42,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:45:43,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:45,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:45:45,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:45:45,457 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:45:48,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:51,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:45:53,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:45:54,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:45:58,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:00,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:46:00,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:46:00,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:46:04,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:46:04,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:09,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:12,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:46:15,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:46:15,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:46:15,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:46:15,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:20,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:21,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:46:21,447 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:46:21,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:46:21,449 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:46:21,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:46:21,805 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:46:22,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:46:25,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:30,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:46:30,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:46:30,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:46:31,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:37,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:38,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:46:42,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:45,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:46:45,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:46:45,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:46:45,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:46:48,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:48,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:46:51,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:46:53,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:58,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:46:58,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:00,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:47:00,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:47:00,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:47:04,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:07,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:10,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:15,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:47:15,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:47:15,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:47:15,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:17,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:20,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:21,797 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:47:25,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:27,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:30,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:47:30,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:47:30,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:47:31,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:36,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:36,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:41,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:45,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:47:45,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:47:45,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:47:46,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:47,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:51,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:47:52,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:47:56,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:47:58,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:00,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:48:00,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:48:00,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:48:03,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:06,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:48:09,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:14,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:15,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:48:15,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:48:15,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:48:16,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:48:19,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:21,800 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:48:24,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:26,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:48:29,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:30,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:48:30,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:48:30,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:48:35,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:36,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:48:40,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:45,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:48:45,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:48:45,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:48:46,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:46,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:48:51,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:48:51,801 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:48:56,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:48:56,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:00,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:49:00,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:49:00,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:49:00,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:49:02,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:06,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:49:08,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:13,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:15,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:49:15,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:49:15,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:49:17,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:49:18,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:21,801 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:49:23,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:27,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:49:29,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:30,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:49:30,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:49:30,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:49:34,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:37,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:49:40,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:44,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:49:44,794 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:49:44,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:49:44,798 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:49:45,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:49:45,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:49:45,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:49:45,459 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:49:45,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:47,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:49:50,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:51,802 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:49:56,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:49:57,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:50:00,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:50:00,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:50:00,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:50:01,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:06,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:50:06,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:11,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:15,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:50:15,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:50:15,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:50:16,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:50:17,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:21,802 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:50:22,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:26,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:50:28,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:30,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:50:30,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:50:30,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:50:30,460 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:50:33,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:36,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:50:38,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:43,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:45,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:50:45,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:50:45,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:50:45,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:50:46,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:50:48,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:51,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:50:54,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:50:58,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:00,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:00,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:51:00,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:51:00,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:51:05,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:08,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:11,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:15,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:51:15,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:51:15,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:51:15,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:51:17,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:18,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:21,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:51:22,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:27,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:27,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:30,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:51:30,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:51:30,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:51:33,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:37,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:38,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:43,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:45,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:51:45,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:51:45,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:51:45,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:51:47,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:49,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:51,805 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:51:54,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:51:57,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:51:59,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:00,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:52:00,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:52:00,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:52:00,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:52:05,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:07,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:52:10,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:15,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:52:15,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:52:15,461 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:52:16,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:17,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:52:21,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:21,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:52:26,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:27,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:52:30,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:52:30,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:52:30,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:52:30,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:52:32,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:37,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:52:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:42,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:45,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:52:45,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:52:45,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:52:47,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:52:48,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:51,807 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:52:53,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:52:57,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:52:59,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:00,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:53:00,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:53:00,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:53:04,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:04,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:53:04,433 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:53:04,434 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:53:04,437 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:53:04,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:53:05,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:06,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:09,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:14,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:15,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:53:15,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:53:15,462 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:53:16,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:19,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:21,807 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:53:25,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:26,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:30,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:30,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:53:30,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:53:30,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:53:30,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:53:35,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:36,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:40,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:44,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:45,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:53:45,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:53:45,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:53:45,463 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:53:45,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:50,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:53:51,808 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:53:54,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:53:55,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:00,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:54:00,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:54:00,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:54:00,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:54:00,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:05,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:54:06,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:11,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:15,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:54:15,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:54:15,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:54:15,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:54:16,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:21,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:21,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:54:25,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:54:26,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:30,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:54:30,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:54:30,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:54:30,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:54:32,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:35,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:54:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:44,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:45,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:54:45,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:54:45,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:54:45,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:54:49,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:54:51,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:54:55,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:54:55,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:00,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:01,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:55:01,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:55:01,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:55:01,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:55:05,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:55:06,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:11,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:15,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:55:16,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:55:16,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:55:16,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:55:16,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:21,812 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:55:22,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:25,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:55:27,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:31,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:55:31,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:55:31,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:55:31,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:55:33,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:35,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:55:38,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:43,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:55:44,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:46,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:55:46,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:55:46,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:55:49,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:55:51,813 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:55:53,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:55:55,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:00,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:01,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:56:01,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:56:01,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:56:01,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:56:02,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:05,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:10,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:14,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:15,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:16,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:56:16,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:56:16,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:56:20,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:21,814 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:56:22,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:56:22,553 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:56:22,554 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:56:22,556 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:56:22,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:56:24,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:25,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:26,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:31,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:31,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:56:31,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:56:31,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:56:33,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:37,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:42,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:43,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:46,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:56:46,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:56:46,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:56:48,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:51,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:56:53,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:56:54,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:56:59,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:01,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:57:01,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:57:01,772 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:57:05,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:57:05,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:10,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:15,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:15,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:57:16,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:57:16,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:57:16,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:57:20,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:21,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:57:25,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:57:26,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:31,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:32,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:57:32,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:57:32,040 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:57:35,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:57:36,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:41,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:45,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:57:47,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:57:47,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:57:47,040 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:57:47,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:51,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:57:52,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:57:55,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:57:58,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:02,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:58:02,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:58:02,040 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:58:03,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:05,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:58:08,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:13,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:15,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:58:17,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:58:17,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:58:17,040 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:58:19,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:21,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:58:23,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:58:24,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:29,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:32,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:58:32,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:58:32,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:58:32,042 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:58:33,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:58:35,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:40,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:45,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:58:45,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:47,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:58:47,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:58:47,040 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:58:51,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:58:51,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:58:55,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:58:57,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:02,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:59:02,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:59:02,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:59:02,044 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:59:02,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:05,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:59:07,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:12,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:15,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:59:17,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:59:17,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:59:17,040 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:59:18,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:21,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:59:23,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:25,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:59:28,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:32,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:59:32,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:59:32,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:59:32,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:59:33,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:35,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:59:38,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:43,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 11:59:43,576 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 11:59:43,576 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 11:59:43,579 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 11:59:44,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 11:59:44,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:45,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 11:59:47,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 11:59:47,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 11:59:47,580 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 11:59:49,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:51,822 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 11:59:55,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 11:59:55,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:00,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:02,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:00:02,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:00:02,583 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:00:05,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:05,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:10,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:15,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:16,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:17,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:00:17,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:00:17,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:00:21,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:21,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:00:25,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:27,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:32,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:32,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:00:32,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:00:32,585 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:00:35,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:37,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:42,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:46,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:47,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:00:47,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:00:47,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:00:47,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:51,824 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:00:52,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:00:56,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:00:58,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:02,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:01:02,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:01:02,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:01:03,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:08,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:12,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:01:14,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:18,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:01:18,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:01:18,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:01:18,492 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:01:19,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:21,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:01:21,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:01:25,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:30,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:31,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:01:33,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:01:33,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:01:36,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:41,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:01:41,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:46,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:48,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:01:48,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:01:48,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:01:48,795 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:01:51,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:01:51,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:01:52,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:01:57,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:01,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:02:03,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:03,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:02:03,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:02:08,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:11,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:02:14,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:18,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:02:18,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:02:18,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:02:18,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:02:20,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:21,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:02:21,826 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:02:25,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:31,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:31,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:02:33,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:02:33,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:02:33,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:02:37,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:41,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:02:42,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:47,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:48,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:02:48,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:02:48,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:02:51,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:02:51,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:02:52,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:02:57,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:02,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:03:03,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:03,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:03:03,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:03:03,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:03:09,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:09,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:03:09,827 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:03:09,828 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:03:09,831 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:03:10,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:03:12,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:03:14,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:18,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:03:18,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:03:18,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:03:20,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:21,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:03:22,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:03:25,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:31,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:32,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:03:33,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:03:33,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:03:33,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:03:37,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:40,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:03:42,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:47,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:48,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:03:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:03:48,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:03:50,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:03:51,828 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:03:52,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:57,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:03:59,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:04:02,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:03,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:04:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:04:03,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:04:08,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:09,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:04:14,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:18,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:04:18,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:04:18,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:04:18,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:04:19,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:19,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:04:21,828 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:04:24,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:29,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:04:30,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:33,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:04:33,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:04:33,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:04:36,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:39,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:04:41,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:46,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:04:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:04:48,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:04:51,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:51,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:04:51,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:04:56,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:04:59,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:05:01,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:05:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:05:03,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:05:07,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:10,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:05:12,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:17,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:18,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:05:18,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:05:18,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:05:18,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:05:20,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:05:21,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:05:22,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:27,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:30,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:05:33,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:33,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:05:33,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:05:33,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:05:38,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:40,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:05:43,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:05:48,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:05:48,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:05:49,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:05:50,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:05:51,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:05:54,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:00,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:00,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:06:03,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:06:03,885 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:06:06,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:10,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:11,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:16,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:18,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:06:18,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:06:18,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:06:20,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:21,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:06:22,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:22,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:27,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:31,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:06:31,595 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:06:31,595 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:06:31,598 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:06:32,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:06:32,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:33,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:33,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:06:33,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:06:33,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:06:34,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:39,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:44,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:44,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:48,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:06:48,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:06:48,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:06:48,888 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:06:50,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:06:51,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:06:53,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:06:55,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:00,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:03,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:07:03,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:07:03,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:07:03,886 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:07:06,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:11,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:13,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:07:16,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:20,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:07:20,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:07:20,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:07:20,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:07:21,847 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:07:21,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:21,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:07:26,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:31,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:32,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:07:35,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:07:35,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:07:35,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:07:37,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:42,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:07:43,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:48,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:50,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:07:50,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:07:50,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:07:50,242 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:07:51,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:07:52,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:07:53,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:07:58,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:02,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:08:04,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:05,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:08:05,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:08:05,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:08:09,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:12,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:08:14,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:20,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:08:20,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:08:20,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:08:20,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:21,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:08:22,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:08:25,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:30,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:08:31,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:35,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:08:35,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:08:35,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:08:36,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:40,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:08:41,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:46,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:50,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:08:50,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:08:50,241 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:08:50,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:08:51,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:08:52,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:08:57,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:00,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:03,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:05,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:09:05,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:09:05,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:09:09,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:10,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:14,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:20,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:20,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:09:20,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:09:20,241 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:09:20,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:21,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:09:25,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:29,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:30,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:35,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:36,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:09:36,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:09:36,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:09:36,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:09:39,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:41,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:46,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:48,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:09:48,343 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:09:48,344 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:09:48,346 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:09:49,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:09:50,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:51,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:09:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:09:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:09:51,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:09:51,852 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:09:51,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:09:56,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:00,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:02,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:06,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:10:06,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:10:06,682 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:10:07,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:08,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:12,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:18,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:18,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:21,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:10:21,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:10:21,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:10:23,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:28,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:28,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:33,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:36,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:10:36,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:10:36,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:10:38,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:38,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:43,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:48,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:49,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:10:51,679 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:10:51,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:10:54,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:10:58,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:10:59,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:04,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:06,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:11:06,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:11:06,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:11:08,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:11:09,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:14,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:18,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:11:20,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:21,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:11:21,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:11:21,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:11:25,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:28,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:11:31,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:36,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:36,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:11:36,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:11:36,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:11:38,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:11:41,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:47,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:48,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:11:51,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:11:51,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:11:51,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:11:51,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:11:52,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:58,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:11:58,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:03,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:06,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:12:06,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:12:06,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:12:08,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:08,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:13,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:18,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:19,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:21,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:12:21,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:12:21,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:12:22,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:12:25,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:26,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:30,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:35,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:36,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:36,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:12:36,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:12:36,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:12:41,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:46,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:47,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:51,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:12:51,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:12:51,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:12:51,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:12:52,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:12:56,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:12:58,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:03,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:05,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:13:05,755 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:13:05,756 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:13:05,759 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:13:06,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:13:06,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:13:06,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:06,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:06,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:13:06,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:13:08,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:16,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:13:19,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:21,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:21,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:21,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:13:21,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:13:21,937 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:13:24,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:26,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:13:30,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:35,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:36,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:13:36,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:36,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:36,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:13:36,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:13:40,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:46,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:46,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:13:51,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:13:51,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:51,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:13:51,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:13:51,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:13:51,914 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:13:56,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:13:57,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:02,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:06,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:06,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:06,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:14:06,689 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:14:07,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:14:07,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:12,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:17,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:14:18,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:21,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:21,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:21,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:14:21,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:14:21,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:14:23,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:27,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:14:29,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:36,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:36,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:14:36,690 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:14:37,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:14:40,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:46,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:14:46,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:51,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:14:51,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:51,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:14:51,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:14:51,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:14:51,865 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:14:56,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:14:57,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:02,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:06,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:15:06,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:15:06,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:15:06,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:15:07,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:12,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:16,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:15:18,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:21,858 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:15:23,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:15:23,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:15:23,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:15:23,166 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:15:23,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:26,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:15:29,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:34,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:36,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:15:38,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:15:38,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:15:38,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:15:39,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:44,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:50,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:15:50,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:15:51,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:15:53,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:15:53,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:15:53,165 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:15:55,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:00,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:16:00,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:05,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:08,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:16:08,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:16:08,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:16:08,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:16:10,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:16:11,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:16,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:20,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:16:21,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:16:22,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:23,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:16:23,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:16:23,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:16:27,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:28,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:16:28,900 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:16:28,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:16:28,903 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:16:29,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:16:31,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:16:32,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:37,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:38,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:16:38,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:16:38,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:16:38,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:16:41,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:16:43,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:48,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:51,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:16:51,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:16:53,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:16:53,604 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:16:53,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:16:59,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:01,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:17:04,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:08,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:17:08,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:17:08,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:17:08,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:17:09,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:11,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:17:14,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:20,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:21,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:17:21,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:17:23,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:17:23,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:17:25,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:30,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:17:31,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:36,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:38,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:17:38,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:17:38,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:17:38,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:17:40,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:17:41,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:46,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:50,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:17:51,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:17:52,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:17:53,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:17:53,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:17:58,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:00,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:03,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:08,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:08,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:18:08,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:18:10,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:13,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:18,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:18,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:21,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:18:23,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:18:23,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:18:24,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:28,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:30,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:35,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:38,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:18:38,605 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:18:39,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:40,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:45,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:49,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:51,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:18:51,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:18:55,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:18:55,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:18:55,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:18:55,142 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:18:57,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:18:57,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:02,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:07,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:09,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:19:10,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:19:10,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:19:10,140 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:19:13,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:18,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:19,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:19:21,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:19:23,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:25,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:19:25,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:19:25,143 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:19:29,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:29,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:19:34,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:37,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:19:39,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:40,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:19:40,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:19:40,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:19:45,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:45,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:19:45,297 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:19:45,298 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:19:45,300 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:19:45,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:19:47,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:19:50,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:51,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:19:55,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:19:55,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:19:55,142 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:19:55,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:19:56,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:00,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:05,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:06,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:10,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:20:10,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:20:10,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:20:11,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:16,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:16,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:21,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:21,870 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:20:24,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:25,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:20:25,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:20:25,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:20:27,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:32,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:34,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:37,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:40,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:20:40,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:20:40,141 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:20:43,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:45,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:48,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:51,872 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:20:54,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:20:55,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:20:55,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:20:55,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:20:55,142 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:21:00,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:05,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:21:06,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:11,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:11,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:21:11,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:21:11,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:21:11,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:21:15,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:21:16,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:21,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:21,872 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:21:25,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:21:26,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:21:26,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:21:26,691 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:21:26,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:31,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:33,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:21:37,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:41,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:21:41,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:21:41,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:21:41,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:21:42,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:43,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:21:47,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:51,873 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:21:52,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:21:53,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:21:56,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:21:56,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:21:56,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:21:57,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:03,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:03,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:22:08,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:11,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:22:11,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:22:11,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:22:13,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:22:13,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:18,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:21,873 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:22:23,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:22:24,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:26,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:22:26,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:22:26,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:22:29,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:32,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:22:35,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:40,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:41,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:22:41,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:22:41,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:22:42,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:22:46,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:51,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:22:52,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:22:53,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:22:56,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:22:56,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:22:56,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:22:57,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:00,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:23:00,451 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:23:00,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:23:00,454 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:23:01,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:23:01,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:03,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:03,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:08,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:11,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:11,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:23:11,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:23:11,692 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:23:13,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:18,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:21,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:21,877 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:23:24,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:26,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:23:26,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:23:26,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:23:29,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:31,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:35,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:41,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:41,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:41,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:23:41,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:23:41,693 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:23:46,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:51,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:23:51,878 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:23:52,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:23:56,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:23:56,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:23:56,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:23:57,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:01,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:24:03,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:08,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:11,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:24:11,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:24:11,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:24:11,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:24:14,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:19,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:21,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:24:21,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:24:24,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:26,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:24:26,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:24:26,697 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:24:29,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:31,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:24:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:40,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:41,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:24:41,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:24:41,694 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:24:41,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:24:45,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:51,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:51,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:24:51,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:24:56,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:24:58,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:24:58,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:24:58,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:24:58,148 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:25:01,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:25:02,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:07,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:11,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:25:13,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:25:13,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:25:13,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:25:13,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:18,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:21,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:25:21,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:25:23,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:28,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:25:28,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:25:28,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:25:29,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:31,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:25:34,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:40,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:41,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:25:43,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:25:43,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:25:43,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:25:45,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:50,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:51,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:25:51,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:25:55,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:25:58,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:25:58,251 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:26:01,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:26:01,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:06,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:11,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:26:11,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:13,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:26:13,248 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:26:17,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:19,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:26:19,729 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:26:19,730 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:26:19,733 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:26:20,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:26:21,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:26:21,881 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:26:22,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:27,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:29,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:26:29,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:26:29,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:26:29,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:26:33,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:33,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:26:38,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:43,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:26:43,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:44,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:26:44,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:26:44,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:26:49,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:51,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:26:51,892 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:26:54,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:26:59,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:26:59,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:26:59,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:26:59,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:27:00,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:01,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:27:05,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:11,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:11,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:27:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:27:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:27:14,523 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:27:16,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:21,882 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:27:21,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:21,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:27:26,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:29,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:27:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:27:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:27:29,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:27:32,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:32,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:27:37,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:42,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:27:42,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:44,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:27:44,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:27:44,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:27:47,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:51,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:27:51,883 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:27:52,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:57,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:27:59,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:27:59,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:27:59,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:28:01,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:28:03,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:08,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:11,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:28:13,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:14,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:28:14,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:28:14,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:28:19,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:21,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:28:21,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:28:24,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:29,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:28:29,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:28:29,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:28:30,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:31,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:28:35,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:41,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:41,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:28:44,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:28:44,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:28:44,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:28:46,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:49,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:28:51,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:51,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:28:56,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:28:59,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:28:59,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:28:59,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:28:59,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:29:02,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:07,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:12,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:14,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:29:14,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:29:14,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:29:17,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:19,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:29:21,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:29:22,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:27,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:29,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:29:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:29:29,524 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:29:29,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:29:33,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:38,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:39,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:29:43,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:44,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:29:44,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:29:44,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:29:44,529 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:29:47,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:29:47,516 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:29:47,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:29:47,519 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:29:48,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:29:49,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:50,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:29:51,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:29:54,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:29:59,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:29:59,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:29:59,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:30:00,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:30:00,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:05,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:10,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:30:11,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:14,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:30:14,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:30:14,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:30:14,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:30:16,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:20,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:30:21,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:30:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:26,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:29,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:30:29,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:30:29,525 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:30:30,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:30:32,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:38,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:39,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:30:43,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:44,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:30:44,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:30:44,527 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:30:48,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:51,886 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:30:53,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:30:53,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:30:58,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:00,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:31:00,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:31:00,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:31:00,580 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:31:03,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:31:03,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:08,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:13,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:31:14,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:15,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:31:15,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:31:15,589 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:31:19,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:21,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:31:23,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:31:25,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:30,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:31,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:31:31,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:31:31,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:31:31,433 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:31:33,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:31:35,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:40,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:43,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:31:46,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:31:46,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:31:46,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:31:46,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:51,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:31:51,888 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:31:53,962 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:31:56,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:01,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:32:01,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:32:01,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:32:01,434 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:32:02,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:04,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:32:07,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:13,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:14,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:32:16,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:32:16,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:32:16,430 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:32:18,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:21,889 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:32:24,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:32:24,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:29,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:31,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:32:31,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:32:31,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:32:34,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:32:35,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:40,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:44,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:32:45,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:46,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:32:46,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:32:46,431 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:32:51,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:32:51,890 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:32:54,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:32:57,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:01,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:33:01,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:33:01,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:33:01,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:33:02,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:04,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:07,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:12,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:12,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:33:12,815 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:33:12,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:33:12,819 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:33:13,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:33:13,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:14,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:16,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:33:16,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:33:16,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:33:17,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:21,890 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:33:23,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:23,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:28,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:31,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:33:31,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:33:31,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:33:33,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:34,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:39,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:43,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:45,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:46,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:33:46,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:33:46,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:33:50,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:33:51,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:33:53,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:33:56,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:01,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:01,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:34:01,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:34:01,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:34:01,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:34:03,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:34:07,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:12,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:14,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:34:16,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:34:16,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:34:18,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:21,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:34:23,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:24,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:34:28,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:31,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:34:31,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:34:31,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:34:34,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:34:34,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:39,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:44,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:34:45,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:46,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:34:46,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:34:50,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:34:51,892 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:34:54,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:34:55,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:00,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:01,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:35:01,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:35:04,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:35:06,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:12,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:14,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:35:16,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:35:16,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:35:17,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:21,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:35:22,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:35:22,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:27,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:31,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:35:31,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:35:31,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:35:31,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:35:32,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:34,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:35:37,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:43,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:44,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:35:46,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:35:46,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:35:48,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:51,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:35:53,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:35:53,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:35:58,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:01,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:36:01,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:36:01,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:36:03,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:36:04,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:09,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:13,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:36:14,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:16,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:36:16,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:36:19,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:21,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:36:22,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:36:24,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:29,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:31,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:36:31,330 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:36:31,330 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:36:31,333 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:36:31,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:36:31,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:36:32,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:36:32,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:36:34,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:36:35,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:40,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:45,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:46,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:36:46,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:36:50,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:36:51,896 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:36:55,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:00,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:01,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:37:01,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:37:06,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:11,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:16,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:37:16,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:37:17,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:21,897 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:37:22,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:27,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:31,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:37:31,563 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:37:33,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:38,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:43,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:46,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:37:46,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:37:49,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:51,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:37:54,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:37:59,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:01,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:38:01,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:38:05,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:10,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:15,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:16,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:38:16,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:38:20,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:21,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:38:25,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:30,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:31,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:38:31,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:38:36,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:41,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:46,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:38:46,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:38:47,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:51,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:38:52,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:38:57,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:01,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:39:01,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:39:03,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:08,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:13,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:16,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:39:16,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:39:18,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:21,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:39:23,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:28,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:31,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:39:31,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:39:34,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:39,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:44,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:46,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:39:46,565 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:39:49,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:51,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:39:54,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:39:59,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:01,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:40:01,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:40:01,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:40:05,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:10,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:15,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:16,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:40:16,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:40:16,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:40:21,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:21,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:40:26,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:31,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:40:31,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:40:32,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:37,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:42,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:46,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:40:46,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:40:48,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:51,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:40:53,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:40:58,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:01,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:41:01,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:41:01,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:41:04,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:09,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:14,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:16,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:41:16,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:41:16,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:41:19,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:21,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:41:24,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:29,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:31,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:41:31,566 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:41:31,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:41:35,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:40,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:45,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:46,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:41:46,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:41:51,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:41:51,901 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:41:56,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:01,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:42:01,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:42:01,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:06,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:11,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:16,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:42:16,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:42:17,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:21,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:42:22,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:27,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:31,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:42:31,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:42:33,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:38,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:43,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:46,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:42:46,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:42:48,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:51,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:42:53,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:42:58,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:01,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:43:01,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:43:04,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:09,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:14,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:16,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:43:16,567 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:43:19,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:21,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:43:24,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:29,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:31,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:43:31,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:43:35,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:40,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:45,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:46,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:43:46,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:43:51,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:43:51,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:43:56,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:01,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:44:01,568 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:44:02,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:07,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:12,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:16,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:44:16,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:44:16,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:44:17,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:21,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:44:22,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:27,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:31,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:44:31,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:44:33,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:38,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:43,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:46,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:44:46,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:44:49,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:51,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:44:54,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:44:59,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:01,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:45:01,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:45:05,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:10,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:15,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:16,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:45:16,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:45:20,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:21,904 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:45:25,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:30,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:31,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:45:31,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:45:36,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:41,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:46,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:45:46,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:45:47,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:51,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:45:52,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:45:57,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:01,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:46:01,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:46:03,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:08,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:13,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:16,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:46:16,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:46:18,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:21,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:46:23,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:28,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:31,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:46:31,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:46:34,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:39,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:44,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:46,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:46:46,570 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:46:50,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:46:51,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:46:55,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:00,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:01,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:47:01,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:47:01,574 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:47:06,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:11,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:16,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:47:16,500 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:47:16,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:47:16,503 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:47:16,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:47:16,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:47:16,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:47:17,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:47:17,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:19,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:47:21,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:47:22,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:27,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:31,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:47:31,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:47:31,571 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:47:33,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:38,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:44,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:45,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:47:46,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:47:46,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:47:46,935 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:47:50,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:47:51,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:47:55,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:00,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:01,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:48:02,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:48:02,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:48:02,043 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:48:06,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:11,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:16,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:17,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:48:17,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:48:21,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:21,909 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:48:26,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:31,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:32,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:48:33,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:48:33,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:48:37,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:42,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:47,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:48,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:48:48,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:48:48,479 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:48:50,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:48:52,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:48:53,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:53,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:48:58,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:48:58,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:49:03,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:08,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:13,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:18,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:19,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:49:22,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:49:24,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:29,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:31,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:49:31,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:49:31,427 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:49:33,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:49:34,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:49:34,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:49:34,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:39,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:44,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:46,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:49:49,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:49:49,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:49:50,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:49:52,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:49:56,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:01,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:50:01,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:04,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:50:04,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:50:06,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:11,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:16,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:50:16,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:19,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:50:19,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:50:21,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:22,459 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:50:27,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:31,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:50:32,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:34,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:50:34,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:50:35,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:50:37,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:42,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:46,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:50:47,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:49,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:50:49,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:50:49,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:50:52,459 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:50:53,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:50:58,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:01,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:51:03,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:04,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:51:04,561 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:51:08,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:13,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:15,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:51:16,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:51:19,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:19,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:51:19,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:51:22,461 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:51:24,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:27,844 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:51:29,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:51:34,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:51:34,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:51:34,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:37,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:51:40,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:45,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:46,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:51:47,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:51:49,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:51:49,562 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:51:50,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:52,462 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:51:56,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:51:57,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:52:01,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:01,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:52:05,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:52:05,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:52:06,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:07,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:52:11,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:16,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:52:16,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:17,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:52:20,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:52:20,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:52:22,462 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:52:22,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:27,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:52:27,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:52:32,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:35,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:52:35,391 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:52:37,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:39,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:52:42,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:46,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:52:47,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:49,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:52:50,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:52:50,388 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:52:52,463 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:52:53,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:58,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:52:59,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:53:01,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:53:03,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:06,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:53:06,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:53:09,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:09,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:53:14,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:16,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:53:19,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:19,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:53:21,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:53:21,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:53:22,463 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:53:25,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:29,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:53:30,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:31,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:53:35,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:36,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:53:36,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:53:39,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:53:40,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:45,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:46,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:53:50,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:53:51,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:53:51,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:53:51,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:53:52,464 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:53:57,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:01,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:54:02,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:54:02,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:06,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:54:06,316 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:54:08,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:11,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:54:14,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:18,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:54:19,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:19,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:54:19,471 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:54:19,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:54:19,475 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:54:20,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:54:21,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:54:21,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:54:21,313 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:54:22,480 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:54:24,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:29,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:31,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:54:33,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:54:35,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:36,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:54:36,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:54:40,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:41,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:54:45,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:49,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:54:50,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:54:51,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:54:51,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:54:51,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:54:52,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:54:56,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:01,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:55:01,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:04,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:55:06,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:55:06,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:55:07,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:11,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:55:12,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:17,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:19,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:55:21,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:55:21,314 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:55:22,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:55:23,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:23,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:55:28,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:33,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:33,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:55:34,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:55:36,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:55:36,315 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:55:38,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:43,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:55:44,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:49,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:50,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:55:51,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:55:51,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:55:52,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:55:54,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:55:54,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:55:59,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:04,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:56:04,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:05,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:56:06,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:56:06,316 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:56:10,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:14,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:56:16,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:21,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:56:21,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:56:21,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:56:21,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:22,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:56:24,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:56:27,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:32,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:33,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:56:36,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:56:36,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:56:36,316 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:56:37,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:42,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:45,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:56:48,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:51,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:56:52,469 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:56:52,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:56:52,761 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:56:54,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:56:55,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:56:59,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:04,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:05,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:57:06,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:57:07,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:57:07,771 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:57:09,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:15,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:57:15,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:20,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:21,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:57:22,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:57:23,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:57:23,892 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:57:26,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:27,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:57:31,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:36,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:57:36,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:37,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:57:38,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:57:38,889 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:57:42,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:46,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 12:57:46,316 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 12:57:46,317 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 12:57:46,320 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 12:57:46,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 12:57:47,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:47,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:57:51,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:57:52,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:52,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:57:54,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:57:54,985 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:57:57,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:57:58,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:58:02,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:06,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:58:08,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:58:08,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:09,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:58:09,984 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:58:13,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:18,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:20,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:58:21,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:58:22,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:58:23,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:26,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:58:26,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:58:26,096 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:58:29,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:30,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:58:34,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:36,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:58:39,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:40,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:58:41,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:58:41,096 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:58:45,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:49,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:58:50,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:51,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:58:52,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:58:56,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:58:56,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:58:56,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:58:56,822 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:58:59,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:59:02,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:07,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:07,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:09,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:59:11,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:59:11,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:59:13,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:18,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:19,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:59:22,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:59:22,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:23,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:28,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:28,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:59:28,004 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:59:29,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:29,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:59:34,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:37,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:39,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:59:40,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:43,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:43,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:59:43,003 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 12:59:45,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:50,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 12:59:50,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:52,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 12:59:52,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:55,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 12:59:58,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 12:59:58,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 12:59:58,532 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:00:01,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:02,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:00:06,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:07,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:11,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:13,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:13,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:00:13,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:00:16,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:00:17,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:22,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:22,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:00:24,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:26,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:00:27,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:28,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:00:28,797 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:00:32,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:38,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:00:38,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:39,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:43,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:44,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:44,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:00:44,972 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:00:48,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:00:49,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:52,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:00:54,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:54,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:59,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:00:59,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:00:59,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:00:59,971 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:01:00,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:05,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:09,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:10,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:10,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:14,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:14,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:01:14,971 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:01:16,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:19,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:01:19,264 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:01:19,264 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:01:19,268 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:01:19,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:01:19,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:21,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:21,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:22,475 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:01:24,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:26,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:29,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:29,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:29,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:01:29,972 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:01:32,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:37,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:39,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:41,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:42,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:44,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:44,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:01:44,972 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:01:48,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:52,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:01:53,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:01:54,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:01:54,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:01:58,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:00,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:01,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:02:01,060 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:02:04,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:04,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:02:09,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:09,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:14,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:02:14,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:15,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:15,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:02:15,700 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:02:19,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:22,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:02:24,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:24,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:02:25,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:30,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:31,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:31,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:02:31,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:02:34,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:02:35,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:39,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:40,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:44,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:02:45,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:46,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:46,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:02:46,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:02:51,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:02:52,477 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:02:54,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:02:54,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:02:57,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:01,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:03:01,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:03:01,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:03:03,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:04,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:03:08,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:09,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:03:14,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:14,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:03:16,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:03:16,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:03:19,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:22,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:03:24,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:03:24,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:03:25,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:30,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:31,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:03:31,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:03:34,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:03:36,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:39,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:03:41,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:43,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:03:46,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:46,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:03:46,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:03:51,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:03:52,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:03:53,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:03:54,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:03:57,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:01,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:04:01,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:04:01,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:04:03,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:04,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:04:08,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:09,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:04:14,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:04:14,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:16,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:04:16,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:04:19,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:22,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:04:24,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:04:24,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:04:25,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:30,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:31,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:04:31,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:04:31,382 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:04:34,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:04:36,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:39,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:04:41,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:43,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:04:43,058 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:04:43,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:04:43,061 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:04:43,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:04:44,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:04:46,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:04:46,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:04:46,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:51,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:04:52,482 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:04:54,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:04:54,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:04:57,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:02,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:02,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:05:02,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:05:02,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:04,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:05:08,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:09,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:13,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:14,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:05:17,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:17,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:05:17,087 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:05:19,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:22,493 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:05:24,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:24,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:05:25,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:30,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:32,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:32,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:05:32,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:05:34,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:05:35,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:39,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:40,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:44,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:05:45,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:47,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:47,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:05:47,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:05:51,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:05:52,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:05:54,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:05:54,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:05:57,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:02,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:02,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:06:02,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:06:02,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:04,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:06:08,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:09,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:13,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:15,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:06:17,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:17,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:06:17,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:06:19,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:22,484 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:06:24,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:24,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:06:24,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:29,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:32,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:32,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:06:32,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:06:34,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:06:35,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:39,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:40,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:44,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:06:45,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:47,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:47,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:06:47,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:06:50,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:06:52,485 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:06:54,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:06:54,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:06:56,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:01,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:02,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:02,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:07:02,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:07:04,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:07:06,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:09,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:11,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:14,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:07:17,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:17,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:17,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:07:17,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:07:22,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:22,486 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:07:24,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:24,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:07:28,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:32,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:32,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:07:32,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:07:33,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:34,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:07:38,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:39,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:44,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:07:44,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:47,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:47,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:07:47,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:07:50,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:07:52,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:07:54,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:07:54,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:07:56,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:01,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:02,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:02,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:08:02,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:08:03,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:08:03,419 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:08:03,420 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:08:03,422 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:08:03,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:08:04,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:08:06,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:09,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:11,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:14,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:08:17,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:17,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:08:17,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:08:17,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:22,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:22,489 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:08:24,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:24,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:08:27,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:32,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:32,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:08:32,401 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:08:33,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:35,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:08:38,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:39,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:43,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:47,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:08:47,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:08:47,399 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:08:49,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:52,490 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:08:54,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:08:55,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:08:57,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:00,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:02,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:09:02,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:09:02,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:09:05,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:06,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:09,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:09:11,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:16,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:16,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:17,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:09:17,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:09:17,813 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:09:22,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:22,492 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:09:24,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:09:26,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:27,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:32,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:34,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:09:34,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:09:34,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:09:36,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:38,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:43,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:46,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:48,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:49,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:09:49,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:09:49,074 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:09:52,492 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:09:53,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:09:56,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:09:59,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:04,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:10:04,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:10:04,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:10:04,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:06,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:10:10,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:15,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:16,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:10:19,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:10:19,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:10:19,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:10:20,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:22,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:10:24,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:10:25,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:27,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:10:30,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:34,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:10:34,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:10:34,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:10:36,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:37,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:10:41,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:46,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:47,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:10:49,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:10:49,904 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:10:52,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:52,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:10:55,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:10:57,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:10:57,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:11:02,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:05,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:05,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:11:05,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:11:07,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:09,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:11:12,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:17,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:19,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:11:20,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:20,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:11:20,261 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:11:22,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:11:23,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:25,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:26,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:11:26,882 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:11:26,882 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:11:26,885 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:11:27,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:11:28,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:29,605 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:11:33,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:35,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:35,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:11:35,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:11:39,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:11:39,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:40,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:44,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:49,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:11:50,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:50,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:50,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:11:50,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:11:52,495 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:11:55,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:11:55,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:11:58,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:12:01,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:05,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:12:05,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:12:05,800 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:12:07,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:08,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:12:12,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:17,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:19,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:12:20,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:12:20,799 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:12:22,495 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:12:22,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:28,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:29,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:12:33,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:35,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:12:35,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:12:35,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:12:38,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:39,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:12:43,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:48,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:49,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:12:50,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:12:50,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:12:52,496 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:12:53,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:55,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:12:59,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:12:59,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:13:04,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:05,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:13:05,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:13:09,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:13:10,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:10,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:13:15,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:20,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:20,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:13:20,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:13:21,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:13:22,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:13:25,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:25,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:13:30,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:31,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:13:35,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:35,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:13:35,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:13:40,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:13:41,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:41,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:13:46,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:50,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:13:50,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:13:51,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:13:51,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:13:52,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:13:55,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:13:56,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:01,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:01,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:14:05,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:14:05,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:14:07,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:10,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:14:11,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:14:12,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:17,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:20,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:14:20,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:14:21,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:14:22,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:14:23,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:25,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:14:28,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:33,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:14:33,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:35,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:14:35,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:14:39,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:40,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:14:43,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:14:44,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:49,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:50,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:14:50,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:14:50,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:14:51,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:14:51,168 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:14:51,169 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:14:51,171 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:14:52,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:14:52,498 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:14:53,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:14:55,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:14:55,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:15:00,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:05,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:05,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:15:05,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:15:07,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:15:10,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:15:11,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:16,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:17,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:15:20,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:15:20,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:15:22,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:22,499 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:15:25,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:15:27,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:15:27,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:32,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:35,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:15:35,901 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:15:37,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:15:38,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:40,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:15:43,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:47,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:15:48,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:50,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:15:50,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:15:52,500 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:15:53,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:15:55,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:15:57,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:15:59,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:04,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:05,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:16:05,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:16:05,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:16:07,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:16:09,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:10,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:16:14,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:17,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:16:19,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:20,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:16:20,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:16:22,502 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:16:25,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:25,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:16:25,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:16:30,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:35,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:16:35,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:16:35,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:16:36,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:40,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:16:41,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:45,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:16:46,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:50,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:16:50,902 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:16:52,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:16:52,503 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:16:55,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:16:56,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:16:57,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:02,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:05,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:17:05,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:17:05,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:17:08,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:17:08,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:10,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:17:13,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:16,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:17:18,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:20,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:17:20,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:17:22,504 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:17:24,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:25,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:17:25,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:17:29,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:35,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:17:35,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:17:35,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:17:40,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:17:41,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:45,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:17:46,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:51,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:17:52,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:17:52,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:17:52,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:17:52,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:17:55,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:17:57,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:02,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:05,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:18:07,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:18:07,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:18:07,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:18:07,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:12,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:13,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:18:13,259 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:18:13,260 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:18:13,262 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:18:13,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:18:15,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:18:18,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:22,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:18:22,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:18:22,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:18:22,556 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:18:23,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:25,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:18:28,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:33,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:35,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:18:37,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:18:37,291 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:18:39,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:45,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:45,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:18:50,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:52,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:18:52,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:18:52,293 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:18:52,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:18:55,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:18:55,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:00,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:05,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:05,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:07,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:19:07,292 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:19:11,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:12,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:19:16,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:17,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:22,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:22,509 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:19:23,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:19:23,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:19:23,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:19:26,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:27,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:32,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:36,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:37,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:38,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:19:38,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:19:38,166 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:19:43,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:46,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:48,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:52,518 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:19:53,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:19:53,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:19:53,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:19:54,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:19:56,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:19:59,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:05,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:05,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:20:08,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:20:08,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:20:08,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:20:10,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:15,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:20:16,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:21,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:22,519 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:20:23,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:20:23,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:20:23,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:20:25,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:20:26,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:31,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:35,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:20:37,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:20:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:20:38,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:20:43,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:45,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:20:48,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:52,520 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:20:53,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:20:53,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:20:53,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:20:54,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:20:55,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:20:59,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:04,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:05,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:21:08,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:21:08,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:21:08,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:21:10,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:15,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:15,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:21:20,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:22,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:21:23,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:21:23,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:21:23,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:21:25,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:26,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:21:30,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:33,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:21:33,868 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:21:33,869 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:21:33,871 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:21:34,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:21:35,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:36,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:21:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:21:38,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:21:38,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:21:41,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:46,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:21:47,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:52,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:21:52,523 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:21:53,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:21:53,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:21:53,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:21:56,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:21:57,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:02,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:06,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:22:07,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:08,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:22:08,175 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:22:13,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:13,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:22:16,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:22:18,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:22,523 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:22:23,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:22:23,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:22:23,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:22:24,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:26,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:22:28,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:22:29,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:34,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:38,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:22:38,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:22:38,279 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:22:39,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:43,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:22:44,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:49,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:22:52,524 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:22:53,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:22:53,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:22:53,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:22:53,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:22:55,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:00,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:00,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:03,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:23:05,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:08,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:08,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:23:08,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:23:11,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:14,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:23:15,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:17,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:22,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:22,525 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:23:23,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:23,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:23:23,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:23:24,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:23:27,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:31,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:33,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:34,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:23:38,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:38,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:23:38,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:23:38,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:44,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:23:44,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:46,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:49,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:52,525 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:23:53,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:23:53,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:23:53,280 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:23:54,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:23:54,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:23:59,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:01,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:04,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:24:04,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:08,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:08,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:24:08,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:24:10,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:15,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:16,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:24:16,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:20,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:22,527 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:24:23,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:23,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:24:23,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:24:25,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:26,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:24:30,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:32,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:35,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:36,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:24:38,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:38,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:24:38,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:24:41,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:46,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:24:46,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:47,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:51,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:24:52,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:24:53,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:24:53,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:24:53,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:24:54,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:24:57,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:02,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:02,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:04,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:25:08,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:08,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:25:08,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:25:08,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:13,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:25:13,194 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:25:13,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:25:13,197 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:25:13,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:25:13,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:25:14,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:17,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:19,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:22,529 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:25:23,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:23,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:25:23,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:25:23,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:25:24,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:29,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:32,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:34,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:25:35,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:38,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:38,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:25:38,281 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:25:40,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:44,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:25:45,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:47,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:50,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:25:52,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:25:53,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:25:53,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:25:53,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:25:54,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:25:56,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:01,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:02,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:04,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:26:07,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:08,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:08,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:26:08,282 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:26:12,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:14,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:26:17,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:18,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:22,533 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:26:23,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:23,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:26:23,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:26:23,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:26,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:26:28,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:32,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:33,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:36,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:26:38,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:38,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:26:38,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:26:39,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:44,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:46,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:26:47,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:49,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:52,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:26:53,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:26:53,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:26:53,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:26:55,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:26:56,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:00,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:03,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:06,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:06,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:08,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:08,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:27:08,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:27:11,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:16,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:17,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:18,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:22,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:22,537 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:27:23,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:23,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:27:23,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:27:26,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:28,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:33,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:33,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:36,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:38,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:38,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:38,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:27:38,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:27:43,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:46,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:48,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:48,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:52,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:27:53,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:27:53,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:27:53,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:27:54,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:27:56,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:27:59,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:03,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:04,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:28:05,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:08,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:08,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:28:08,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:28:11,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:14,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:28:17,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:18,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:22,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:22,538 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:28:23,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:23,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:28:23,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:28:24,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:28:27,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:32,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:33,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:33,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:28:33,457 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:28:33,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:28:33,462 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:28:34,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:28:34,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:28:38,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:38,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:28:38,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:28:38,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:44,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:44,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:28:48,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:49,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:52,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:28:53,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:28:53,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:28:53,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:28:54,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:28:54,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:28:59,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:03,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:04,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:29:04,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:08,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:08,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:29:08,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:29:10,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:14,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:29:15,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:18,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:20,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:22,540 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:29:23,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:23,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:29:23,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:29:24,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:29:26,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:31,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:33,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:34,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:29:37,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:38,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:38,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:29:38,284 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:29:42,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:45,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:29:47,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:48,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:52,553 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:29:53,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:29:53,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:29:53,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:29:53,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:29:55,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:29:58,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:03,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:03,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:04,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:30:08,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:08,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:30:08,286 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:30:08,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:13,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:18,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:18,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:30:19,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:22,542 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:30:23,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:23,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:30:23,286 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:30:24,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:28,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:30:30,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:33,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:35,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:38,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:38,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:30:38,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:30:38,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:30:40,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:45,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:48,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:48,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:30:50,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:52,542 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:30:53,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:30:53,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:30:53,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:30:56,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:30:58,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:31:01,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:03,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:06,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:08,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:08,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:31:08,286 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:31:08,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:31:12,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:17,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:18,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:20,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:31:22,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:31:22,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:23,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:23,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:31:23,286 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:31:28,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:30,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:31:33,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:33,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:38,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:38,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:31:38,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:31:38,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:41,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:31:43,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:48,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:48,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:51,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:31:52,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:31:53,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:31:53,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:31:53,286 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:31:54,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:31:58,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:31:58,662 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:31:58,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:31:58,666 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:31:59,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:31:59,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:01,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:32:03,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:04,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:08,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:08,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:32:08,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:32:10,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:11,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:32:15,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:18,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:21,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:21,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:32:22,544 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:32:23,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:23,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:32:23,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:32:26,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:31,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:32:31,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:33,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:36,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:38,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:38,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:32:38,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:32:40,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:32:41,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:46,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:48,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:50,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:32:52,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:32:52,544 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:32:53,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:32:53,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:32:53,287 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:32:57,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:00,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:03,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:03,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:08,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:08,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:08,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:33:08,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:33:10,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:13,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:18,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:19,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:20,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:22,545 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:33:23,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:23,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:33:23,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:33:25,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:30,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:31,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:33,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:36,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:36,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:38,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:38,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:33:38,288 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:33:42,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:46,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:48,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:48,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:52,547 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:33:53,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:33:53,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:33:53,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:33:53,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:33:56,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:33:58,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:03,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:04,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:07,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:34:08,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:34:08,675 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:34:08,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:13,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:19,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:34:19,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:19,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:22,547 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:34:24,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:24,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:24,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:34:24,960 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:34:27,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:34:30,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:34,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:35,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:37,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:34:39,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:34:39,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:39,959 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:34:40,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:45,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:47,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:34:49,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:50,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:52,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:34:54,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:34:54,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:34:54,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:34:56,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:34:57,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:01,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:04,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:07,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:07,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:09,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:35:09,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:09,959 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:35:13,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:15,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:35:15,788 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:35:15,789 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:35:15,791 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:35:16,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:35:16,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:18,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:19,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:22,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:35:24,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:24,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:24,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:35:24,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:35:26,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:30,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:34,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:36,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:36,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:39,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:39,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:35:39,960 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:35:42,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:46,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:47,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:49,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:35:52,552 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:35:53,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:35:54,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:35:54,959 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:35:56,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:35:59,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:04,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:04,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:06,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:36:09,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:09,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:36:09,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:09,960 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:36:14,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:17,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:36:19,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:19,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:22,551 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:36:24,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:36:24,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:24,960 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:36:25,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:27,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:36:30,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:34,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:35,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:36:35,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:39,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:39,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:36:39,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:36:41,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:46,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:47,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:36:49,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:51,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:36:52,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:36:54,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:36:54,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:36:54,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:36:55,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:36:57,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:02,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:04,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:37:07,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:37:07,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:09,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:37:09,960 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:37:13,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:15,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:37:18,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:19,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:37:22,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:37:23,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:24,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:37:24,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:37:24,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:37:25,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:37:29,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:34,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:34,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:37:35,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:37:39,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:39,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:37:39,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:37:45,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:45,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:37:49,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:37:50,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:37:52,556 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:37:54,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:37:54,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:37:54,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:37:54,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:37:56,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:01,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:04,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:38:04,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:06,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:09,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:38:09,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:09,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:38:12,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:14,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:38:17,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:19,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:22,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:22,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:38:24,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:38:24,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:24,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:38:24,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:38:28,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:33,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:33,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:38:33,639 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:38:33,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:38:33,643 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:38:34,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:38:34,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:35,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:38:38,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:39,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:38:39,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:39,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:38:44,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:45,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:38:49,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:38:49,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:52,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:38:54,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:38:54,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:38:54,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:38:55,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:38:55,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:00,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:04,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:39:05,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:39:06,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:09,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:39:09,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:39:09,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:39:11,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:15,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:39:16,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:19,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:39:21,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:22,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:39:24,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:39:24,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:39:25,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:39:27,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:32,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:35,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:39:35,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:39:37,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:39,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:39:39,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:39:43,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:45,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:39:49,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:50,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:39:52,560 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:39:54,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:39:54,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:39:55,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:39:55,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:40:00,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:05,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:40:06,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:07,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:40:09,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:40:09,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:40:11,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:16,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:20,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:40:21,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:22,561 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:40:24,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:40:24,970 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:40:27,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:27,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:40:32,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:35,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:40:37,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:37,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:40:39,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:40:39,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:40:42,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:47,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:40:47,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:50,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:40:52,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:40:53,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:40:54,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:40:54,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:40:55,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:40:59,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:04,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:05,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:41:05,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:41:09,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:09,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:41:09,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:41:14,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:15,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:41:19,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:20,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:41:22,564 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:41:24,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:41:24,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:41:25,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:25,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:41:30,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:35,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:41:35,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:41:35,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:39,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:41:39,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:41:41,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:45,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:41:46,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:50,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:41:51,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:41:52,565 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:41:54,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:41:54,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:41:55,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:41:57,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:02,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:04,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:42:04,425 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:42:04,426 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:42:04,428 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:42:04,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:42:05,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:42:05,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:42:07,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:09,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:42:09,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:42:13,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:15,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:42:19,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:20,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:42:22,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:42:24,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:42:24,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:42:25,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:25,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:42:30,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:35,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:42:35,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:42:35,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:39,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:42:39,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:42:41,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:45,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:42:46,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:50,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:42:51,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:42:52,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:42:54,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:42:54,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:42:55,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:42:57,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:02,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:05,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:43:05,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:43:07,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:09,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:43:09,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:43:13,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:16,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:43:18,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:20,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:43:22,568 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:43:23,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:24,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:43:24,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:43:26,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:43:29,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:34,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:35,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:43:35,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:43:39,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:39,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:43:39,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:43:44,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:45,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:43:49,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:50,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:43:52,570 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:43:54,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:43:54,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:43:55,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:43:55,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:44:00,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:05,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:44:05,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:05,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:44:09,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:44:09,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:44:11,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:15,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:44:16,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:20,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:44:21,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:22,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:44:24,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:44:24,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:44:25,596 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:44:27,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:32,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:35,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:44:35,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:44:37,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:39,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:44:39,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:44:43,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:45,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:44:48,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:50,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:44:52,572 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:44:54,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:44:54,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:44:54,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:44:55,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:00,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:05,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:05,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:45:09,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:45:09,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:45:11,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:11,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:16,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:20,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:45:21,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:22,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:22,572 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:45:24,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:45:24,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:45:27,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:29,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:45:29,225 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:45:29,226 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:45:29,229 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:45:30,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:45:31,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:32,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:32,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:35,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:45:37,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:39,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:45:39,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:45:42,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:43,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:48,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:50,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:45:52,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:45:52,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:45:53,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:45:54,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:45:54,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:45:59,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:02,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:46:04,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:05,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:46:09,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:46:09,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:46:10,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:12,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:46:15,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:20,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:46:20,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:21,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:46:22,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:46:24,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:46:24,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:46:26,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:31,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:46:31,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:35,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:46:36,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:39,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:46:39,966 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:46:41,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:46:42,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:47,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:50,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:46:51,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:46:52,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:52,574 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:46:54,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:46:54,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:46:58,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:46:59,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:47:04,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:05,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:47:09,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:47:09,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:09,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:47:09,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:47:15,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:19,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:47:20,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:47:20,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:22,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:47:24,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:47:24,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:47:26,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:29,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:47:31,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:35,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:47:36,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:39,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:47:39,967 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:47:40,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:47:42,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:47,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:50,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:47:50,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:47:52,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:47:52,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:47:54,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:47:54,970 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:47:58,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:00,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:03,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:06,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:48:09,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:09,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:48:09,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:48:10,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:14,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:19,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:20,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:21,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:48:22,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:48:24,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:48:24,971 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:48:25,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:30,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:31,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:36,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:37,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:48:39,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:48:39,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:48:40,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:41,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:46,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:47,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:48:47,955 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:48:47,956 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:48:47,959 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:48:48,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:48:49,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:50,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:48:51,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:52,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:48:52,577 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:48:56,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:48:56,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:48:57,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:48:59,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:49:02,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:07,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:49:07,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:09,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:49:11,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:49:11,498 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:49:13,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:19,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:19,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:49:22,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:49:22,578 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:49:24,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:26,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:49:26,873 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:49:29,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:49:30,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:35,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:37,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:49:39,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:49:40,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:41,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:49:41,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:49:46,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:48,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:49:51,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:52,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:49:52,578 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:49:56,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:49:56,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:49:56,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:49:58,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:50:02,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:07,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:50:08,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:08,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:50:11,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:50:11,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:50:14,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:18,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:50:20,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:22,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:50:22,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:50:25,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:26,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:50:26,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:50:28,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:50:31,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:36,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:37,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:50:38,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:50:41,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:41,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:50:41,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:50:46,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:48,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:50:51,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:52,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:50:52,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:50:56,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:50:56,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:50:57,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:50:58,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:51:02,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:07,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:51:07,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:08,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:51:11,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:51:11,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:51:13,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:18,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:18,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:51:22,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:51:22,580 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:51:24,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:26,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:51:26,874 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:51:28,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:51:30,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:35,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:37,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:51:38,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:51:40,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:41,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:51:41,874 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:51:46,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:47,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:51:51,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:52,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:51:52,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:51:56,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:51:56,874 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:51:57,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:51:57,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:02,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:05,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:52:05,860 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:52:05,860 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:52:05,864 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:52:06,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:52:07,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:52:07,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:08,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:11,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:52:11,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:52:13,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:16,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:18,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:22,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:52:22,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:52:23,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:26,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:26,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:52:26,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:52:29,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:34,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:36,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:37,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:52:40,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:41,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:52:41,878 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:52:45,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:46,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:50,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:52,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:52:52,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:52:56,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:52:56,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:52:56,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:52:56,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:53:02,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:06,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:53:07,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:07,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:12,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:13,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:13,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:53:13,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:53:16,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:53:18,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:22,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:22,586 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:53:24,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:26,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:53:28,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:28,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:53:28,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:53:29,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:34,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:36,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:53:37,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:39,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:43,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:43,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:53:43,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:53:45,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:46,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:53:50,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:52,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:53:53,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:56,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:53:56,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:53:58,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:53:58,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:53:58,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:54:01,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:06,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:54:07,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:08,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:12,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:13,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:13,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:54:13,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:54:16,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:54:18,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:22,602 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:54:23,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:23,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:27,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:54:28,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:28,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:54:28,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:54:28,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:33,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:37,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:54:38,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:38,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:43,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:43,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:54:43,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:54:44,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:54:44,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:49,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:52,590 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:54:53,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:54,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:54:56,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:54:58,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:54:58,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:54:58,610 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:55:00,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:06,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:06,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:55:08,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:11,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:13,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:13,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:55:13,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:55:16,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:16,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:55:21,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:22,592 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:55:23,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:24,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:55:24,028 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:55:24,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:55:24,032 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:55:24,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:55:26,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:55:27,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:28,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:28,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:55:28,611 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:55:32,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:36,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:55:38,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:38,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:43,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:43,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:43,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:55:43,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:55:46,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:55:49,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:52,593 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:55:53,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:54,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:55:54,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:55:58,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:55:58,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:55:58,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:55:59,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:05,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:06,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:56:08,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:10,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:13,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:13,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:56:13,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:56:16,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:16,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:56:21,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:22,594 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:56:23,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:24,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:56:27,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:28,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:28,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:56:28,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:56:32,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:34,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:56:38,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:38,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:43,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:43,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:56:43,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:56:43,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:45,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:56:48,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:52,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:56:53,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:54,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:56:55,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:56:58,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:56:58,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:56:58,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:56:59,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:05,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:05,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:57:08,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:10,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:13,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:13,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:57:13,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:57:15,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:57:15,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:20,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:22,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:57:23,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:25,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:57:26,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:28,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:28,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:57:28,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:57:31,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:34,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:57:37,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:38,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:42,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:43,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:43,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:57:43,612 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:57:44,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:57:47,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:52,604 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:57:53,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:53,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:54,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:57:58,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:57:58,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:57:58,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:57:58,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:58:03,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:08,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:08,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:13,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:13,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:58:13,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:58:14,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:19,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:22,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:58:23,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:24,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:58:25,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:28,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:28,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:58:28,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:58:30,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:34,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:58:36,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:38,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:41,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:43,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:43,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:58:43,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:58:46,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:58:47,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:52,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:52,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:58:53,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:55,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:58:58,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:58:58,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:58:58,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:58:58,613 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:59:03,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:04,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 13:59:04,380 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 13:59:04,381 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 13:59:04,384 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 13:59:05,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 13:59:06,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:07,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:08,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:09,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:13,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:13,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:59:13,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:59:15,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:20,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:22,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:59:23,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:25,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:25,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:28,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:28,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:59:28,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:59:31,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:35,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:37,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:38,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:42,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:43,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:43,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:59:43,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 13:59:45,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:47,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:52,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:52,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 13:59:53,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:55,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 13:59:58,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 13:59:58,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 13:59:58,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 13:59:58,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:00:03,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:08,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:08,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:10,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:00:13,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:13,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:00:13,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:00:14,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:19,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:22,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:00:23,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:25,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:28,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:28,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:00:28,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:00:30,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:35,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:38,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:40,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:43,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:43,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:00:43,616 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:00:46,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:00:46,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:51,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:52,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:00:53,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:00:56,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:00:57,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:00:58,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:00:58,615 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:01:02,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:06,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:01:08,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:08,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:01:13,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:14,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:01:14,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:01:14,082 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:01:16,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:01:18,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:22,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:01:24,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:24,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:01:27,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:01:29,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:01:29,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:01:30,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:35,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:37,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:01:39,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:01:41,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:44,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:01:44,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:01:44,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:01:47,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:47,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:01:52,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:52,615 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:01:55,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:01:57,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:01:59,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:01:59,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:01:59,701 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:02:02,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:08,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:09,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:02:10,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:02:13,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:14,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:02:14,708 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:02:18,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:19,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:02:22,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:02:24,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:27,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:02:29,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:02:29,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:02:29,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:02:29,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:34,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:39,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:02:40,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:42,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:02:44,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:02:44,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:02:45,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:49,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:02:51,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:52,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:02:57,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:02:57,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:02:58,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:02:58,581 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:02:58,581 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:02:58,584 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:02:58,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:02:59,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:02:59,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:02:59,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:03:00,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:03:02,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:07,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:10,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:03:12,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:03:13,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:14,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:03:14,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:03:18,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:20,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:03:22,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:03:24,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:27,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:03:29,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:03:29,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:03:29,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:30,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:03:34,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:39,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:41,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:03:42,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:03:44,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:03:44,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:03:45,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:51,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:52,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:03:53,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:03:57,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:03:57,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:03:59,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:03:59,703 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:04:02,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:03,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:04:07,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:12,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:04:13,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:04:13,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:14,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:04:14,702 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:04:18,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:22,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:04:23,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:04:24,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:27,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:04:29,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:29,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:04:29,718 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:04:33,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:04:34,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:39,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:41,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:04:42,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:04:44,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:04:44,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:04:44,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:49,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:52,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:04:53,555 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:04:55,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:04:57,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:04:59,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:04:59,718 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:05:00,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:03,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:05:05,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:11,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:12,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:05:13,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:05:14,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:05:14,717 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:05:16,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:22,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:05:22,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:23,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:05:27,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:27,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:05:29,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:05:29,860 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:05:33,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:33,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:05:38,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:42,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:05:42,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:05:43,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:44,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:05:44,858 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:05:49,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:52,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:05:53,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:05:54,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:57,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:05:59,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:05:59,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:05:59,982 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:06:03,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:06:05,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:10,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:12,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:06:13,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:06:14,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:06:14,981 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:06:16,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:21,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:06:21,066 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:06:21,067 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:06:21,070 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:06:21,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:06:22,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:22,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:06:23,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:06:27,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:27,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:06:30,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:06:30,070 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:06:33,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:06:33,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:38,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:42,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:06:43,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:06:43,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:45,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:06:45,070 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:06:49,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:52,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:06:53,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:06:54,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:06:57,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:06:59,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:00,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:07:00,072 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:07:03,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:07:05,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:10,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:12,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:07:13,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:07:15,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:07:15,070 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:07:15,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:20,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:22,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:07:23,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:07:25,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:27,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:07:30,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:07:30,446 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:07:30,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:33,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:07:35,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:40,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:42,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:07:43,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:07:45,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:07:45,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:07:46,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:52,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:52,622 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:07:53,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:07:57,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:07:59,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:08:00,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:08:00,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:08:02,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:04,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:08:07,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:13,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:14,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:08:14,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:08:15,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:08:15,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:08:18,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:22,622 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:08:23,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:08:24,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:29,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:08:30,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:08:30,446 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:08:33,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:08:35,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:40,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:43,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:08:44,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:08:45,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:08:45,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:08:45,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:50,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:52,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:08:53,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:08:56,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:08:59,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:09:00,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:09:00,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:09:01,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:03,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:06,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:11,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:12,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:14,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:09:15,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:09:15,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:09:17,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:21,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:22,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:09:23,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:28,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:29,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:09:30,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:09:30,445 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:09:31,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:33,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:38,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:40,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:09:40,421 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:09:40,422 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:09:40,424 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:09:40,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:09:40,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:41,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:44,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:44,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:09:45,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:09:45,446 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:09:49,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:51,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:09:52,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:09:55,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:09:59,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:10:00,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:10:00,446 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:10:00,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:01,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:10:05,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:10,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:12,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:10:14,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:10:15,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:10:15,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:10:16,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:21,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:22,624 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:10:24,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:10:27,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:29,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:10:30,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:10:30,448 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:10:32,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:10:32,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:37,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:42,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:10:43,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:44,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:10:45,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:10:45,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:10:48,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:52,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:10:54,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:10:54,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:10:59,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:00,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:11:00,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:11:00,449 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:11:03,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:11:05,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:10,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:13,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:11:15,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:11:15,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:11:15,447 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:11:15,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:20,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:22,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:11:23,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:11:26,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:30,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:11:30,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:11:30,449 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:11:32,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:33,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:11:37,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:42,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:43,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:11:45,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:11:45,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:11:45,448 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:11:47,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:52,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:11:53,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:11:53,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:11:58,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:00,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:12:00,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:12:00,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:12:03,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:12:04,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:09,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:14,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:12:15,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:15,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:12:15,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:12:15,448 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:12:20,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:22,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:12:22,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:12:26,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:30,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:12:30,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:12:30,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:12:31,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:32,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:12:36,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:42,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:12:42,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:45,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:12:45,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:12:45,449 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:12:47,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:52,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:12:52,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:12:53,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:12:58,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:00,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:13:00,311 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:13:00,311 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:13:00,314 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:13:00,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:13:00,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:13:00,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:13:00,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:13:01,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:02,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:03,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:08,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:12,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:13,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:15,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:13:15,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:13:15,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:13:19,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:21,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:22,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:13:25,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:30,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:13:30,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:13:30,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:13:30,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:31,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:35,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:41,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:41,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:45,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:13:45,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:13:45,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:13:47,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:51,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:13:52,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:13:53,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:13:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:00,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:14:00,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:14:00,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:14:01,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:14:03,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:08,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:11,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:14:13,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:15,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:14:15,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:14:15,450 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:14:19,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:21,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:14:22,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:14:25,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:30,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:14:30,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:14:30,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:14:30,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:32,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:14:36,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:41,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:42,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:14:45,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:14:45,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:14:45,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:14:47,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:52,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:14:52,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:14:53,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:14:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:00,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:15:00,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:15:00,452 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:15:02,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:15:04,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:09,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:12,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:15:15,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:15,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:15:15,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:15:15,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:15:20,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:22,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:15:26,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:28,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:15:30,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:15:30,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:15:30,452 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:15:31,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:37,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:38,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:15:42,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:45,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:15:45,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:15:45,453 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:15:48,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:48,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:15:52,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:15:53,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:15:58,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:15:59,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:00,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:16:00,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:16:00,452 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:16:04,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:08,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:16:10,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:15,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:16:15,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:16:15,455 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:16:16,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:17,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:16:21,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:22,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:16:26,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:26,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:16:26,755 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:16:26,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:16:26,758 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:16:26,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:16:27,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:16:30,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:16:30,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:16:30,453 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:16:32,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:37,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:16:38,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:43,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:45,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:16:45,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:16:45,454 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:16:48,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:16:48,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:52,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:16:54,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:16:58,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:16:59,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:00,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:17:00,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:17:00,452 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:17:05,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:08,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:17:11,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:16,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:17:16,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:17:16,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:17:16,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:17:16,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:20,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:17:21,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:22,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:17:26,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:30,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:17:31,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:17:31,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:17:31,207 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:17:32,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:38,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:38,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:17:43,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:46,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:17:46,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:17:46,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:17:46,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:17:48,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:17:48,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:52,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:17:54,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:17:58,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:17:59,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:01,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:18:01,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:18:01,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:18:05,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:08,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:18:11,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:16,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:16,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:18:16,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:18:16,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:18:16,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:18:18,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:18:21,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:22,634 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:18:27,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:28,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:18:31,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:18:31,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:18:32,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:38,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:38,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:18:43,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:46,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:18:46,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:18:46,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:18:48,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:48,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:18:52,634 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:18:53,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:18:58,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:18:59,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:01,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:19:01,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:19:04,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:08,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:09,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:14,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:16,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:19:16,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:19:18,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:20,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:22,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:19:25,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:28,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:30,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:31,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:19:31,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:19:36,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:38,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:41,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:45,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:19:45,890 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:19:45,890 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:19:45,893 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:19:46,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:19:46,208 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:19:46,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:19:46,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:47,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:48,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:51,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:19:52,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:19:56,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:19:56,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:01,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:20:01,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:20:02,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:06,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:20:08,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:13,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:16,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:20:16,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:20:16,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:20:18,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:22,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:20:23,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:26,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:20:29,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:31,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:20:31,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:20:34,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:36,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:20:40,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:45,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:46,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:20:46,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:20:46,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:20:51,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:52,641 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:20:56,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:20:56,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:01,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:21:01,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:21:01,213 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:21:02,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:06,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:07,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:12,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:16,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:21:16,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:21:17,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:18,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:22,644 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:21:23,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:27,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:29,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:31,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:21:31,212 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:21:35,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:37,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:40,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:46,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:21:46,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:21:46,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:46,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:51,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:21:52,644 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:21:56,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:21:57,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:02,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:02,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:02,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:02,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:22:02,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:22:06,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:22:07,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:12,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:16,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:22:17,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:17,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:17,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:22:17,755 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:22:18,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:22,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:22:24,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:26,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:22:29,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:33,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:33,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:33,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:22:33,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:22:35,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:36,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:22:40,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:45,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:46,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:22:48,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:48,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:22:48,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:22:48,168 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:22:51,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:22:52,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:22:56,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:22:57,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:02,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:03,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:23:03,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:23:03,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:23:03,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:23:04,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:23:04,555 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:23:04,555 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:23:04,559 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:23:04,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:23:05,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:06,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:07,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:12,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:16,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:18,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:23:18,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:23:18,236 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:23:18,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:22,648 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:23:23,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:26,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:29,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:33,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:23:33,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:23:33,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:23:33,239 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:23:35,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:37,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:40,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:45,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:47,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:48,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:23:48,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:23:48,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:23:50,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:23:52,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:23:55,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:23:55,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:00,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:03,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:24:03,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:24:03,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:24:05,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:24:05,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:10,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:15,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:24:15,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:18,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:24:18,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:24:18,237 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:24:21,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:22,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:24:25,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:24:26,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:31,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:33,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:24:33,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:24:33,239 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:24:36,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:24:37,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:42,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:46,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:24:48,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:48,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:24:48,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:24:48,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:24:52,651 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:24:53,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:24:54,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:24:58,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:03,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:25:03,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:25:03,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:25:04,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:25:04,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:09,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:14,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:25:15,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:18,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:25:18,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:25:18,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:25:21,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:22,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:25:24,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:25:26,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:31,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:33,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:25:33,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:25:33,239 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:25:34,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:25:37,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:42,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:45,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:25:48,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:25:48,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:25:48,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:25:48,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:52,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:25:53,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:25:55,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:25:58,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:03,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:03,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:26:03,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:26:03,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:05,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:07,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:09,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:14,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:17,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:18,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:18,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:26:18,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:18,238 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:26:19,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:22,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:26:24,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:25,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:26:25,272 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:26:25,273 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:26:25,275 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:26:25,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:26:26,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:27,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:30,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:33,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:33,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:33,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:26:33,871 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:26:36,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:37,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:41,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:46,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:47,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:48,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:48,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:26:48,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:26:48,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:26:52,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:26:52,666 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:26:55,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:26:57,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:02,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:03,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:27:03,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:27:03,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:27:05,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:27:07,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:12,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:14,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:27:17,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:18,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:27:18,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:27:18,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:27:22,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:27:22,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:24,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:27:28,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:33,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:27:33,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:27:33,870 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:27:34,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:34,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:27:39,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:44,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:27:44,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:48,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:27:48,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:27:48,871 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:27:50,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:27:52,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:27:54,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:27:55,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:00,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:03,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:28:03,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:28:03,871 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:28:05,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:28:06,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:11,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:15,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:28:16,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:18,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:28:18,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:28:18,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:28:18,872 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:28:22,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:22,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:28:25,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:28:27,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:32,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:33,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:28:33,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:28:33,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:28:33,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:28:35,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:28:38,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:43,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:28:43,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:48,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:48,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:28:48,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:28:48,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:28:52,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:28:53,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:28:53,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:28:58,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:03,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:03,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:29:03,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:29:03,875 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:29:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:09,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:14,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:15,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:18,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:29:18,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:29:18,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:29:18,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:29:20,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:22,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:29:23,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:25,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:33,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:33,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:29:33,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:29:33,876 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:29:36,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:41,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:41,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:29:41,810 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:29:41,811 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:29:41,813 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:29:42,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:29:43,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:44,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:46,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:50,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:29:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:29:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:29:50,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:29:52,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:29:52,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:29:52,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:29:57,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:02,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:05,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:30:05,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:30:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:30:05,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:30:06,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:30:08,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:13,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:18,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:19,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:30:20,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:30:20,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:30:20,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:30:22,658 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:30:24,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:27,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:30:30,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:35,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:30:35,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:30:35,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:30:35,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:37,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:30:41,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:47,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:30:47,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:30:50,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:30:50,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:30:52,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:30:52,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:30:57,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:30:58,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:03,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:05,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:31:05,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:31:05,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:31:07,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:31:08,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:13,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:17,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:31:18,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:20,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:31:20,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:31:20,407 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:31:22,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:31:24,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:27,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:31:30,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:35,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:35,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:31:35,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:31:35,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:31:37,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:31:40,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:45,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:47,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:31:50,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:31:50,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:31:50,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:31:50,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:52,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:31:56,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:31:57,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:32:01,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:05,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:32:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:32:05,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:32:06,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:07,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:32:11,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:17,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:17,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:32:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:32:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:32:20,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:32:22,662 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:32:22,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:27,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:27,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:32:32,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:35,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:32:35,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:32:35,408 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:32:37,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:32:38,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:43,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:47,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:32:49,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:50,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:32:50,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:32:50,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:32:52,664 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:32:54,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:32:57,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:00,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:33:05,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:33:05,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:33:05,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:33:05,648 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:33:05,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:33:05,651 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:33:06,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:33:06,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:06,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:07,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:11,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:16,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:17,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:33:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:33:20,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:33:22,664 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:33:22,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:27,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:27,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:32,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:35,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:33:35,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:33:35,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:33:35,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:38,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:43,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:45,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:48,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:50,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:33:50,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:33:50,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:33:52,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:33:54,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:33:55,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:33:59,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:05,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:34:05,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:34:05,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:34:05,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:34:10,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:15,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:34:16,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:20,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:34:20,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:34:20,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:34:21,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:22,670 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:34:25,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:34:27,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:32,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:35,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:34:35,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:34:35,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:34:35,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:34:37,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:42,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:46,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:34:48,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:50,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:34:50,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:34:50,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:34:52,672 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:34:53,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:34:56,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:34:59,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:05,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:05,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:35:05,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:35:05,410 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:35:06,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:35:10,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:15,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:35:15,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:20,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:35:20,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:35:20,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:35:21,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:22,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:35:25,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:35:27,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:32,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:35,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:35:35,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:35:35,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:35:35,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:35:37,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:42,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:45,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:35:48,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:51,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:35:51,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:35:51,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:35:51,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:35:52,674 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:35:54,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:35:55,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:35:59,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:03,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:36:04,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:06,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:36:06,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:36:06,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:36:10,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:13,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:36:15,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:20,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:21,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:36:21,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:36:22,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:36:22,499 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:36:22,500 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:36:22,503 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:36:22,674 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:36:22,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:36:23,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:36:24,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:36:26,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:31,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:33,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:36:36,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:36:36,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:36:36,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:36:37,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:42,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:45,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:36:47,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:51,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:36:51,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:36:52,674 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:36:52,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:36:57,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:02,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:03,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:37:06,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:37:06,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:37:08,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:13,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:14,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:37:18,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:21,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:37:21,961 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:37:22,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:37:24,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:37:24,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:29,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:34,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:37:35,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:36,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:37:36,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:37:36,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:37:41,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:44,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:37:46,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:51,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:37:51,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:37:51,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:37:52,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:37:52,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:37:54,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:37:57,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:02,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:38:03,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:06,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:38:06,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:38:06,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:38:08,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:13,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:13,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:38:18,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:21,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:38:21,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:38:21,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:38:22,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:38:23,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:28,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:31,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:38:33,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:36,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:38:36,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:38:39,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:41,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:38:44,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:49,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:51,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:38:51,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:38:51,962 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:38:52,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:38:54,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:38:59,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:01,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:05,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:06,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:39:06,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:39:06,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:39:06,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:39:11,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:11,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:16,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:21,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:21,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:39:21,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:39:21,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:22,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:39:27,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:32,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:32,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:36,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:39:36,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:39:36,965 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:39:36,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:39:38,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:42,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:48,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:50,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:39:50,101 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:39:50,102 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:39:50,104 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:39:50,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:39:51,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:51,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:39:51,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:39:52,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:39:52,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:39:54,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:39:59,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:02,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:40:04,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:06,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:40:06,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:40:10,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:12,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:40:15,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:21,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:21,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:40:21,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:40:22,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:40:22,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:40:26,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:31,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:32,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:40:36,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:38,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:40:38,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:40:38,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:40:38,421 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:40:42,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:40:42,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:47,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:52,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:40:52,679 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:40:52,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:40:53,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:40:53,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:40:53,419 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:40:58,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:01,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:41:04,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:08,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:08,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:08,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:41:08,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:41:09,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:11,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:41:14,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:19,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:21,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:41:22,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:41:23,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:23,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:23,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:41:23,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:41:25,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:31,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:31,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:41:36,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:38,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:41:38,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:41:41,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:41,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:41:46,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:52,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:41:52,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:41:52,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:41:53,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:41:53,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:41:53,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:41:57,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:02,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:42:03,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:08,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:08,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:42:08,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:42:08,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:42:08,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:42:12,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:42:14,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:20,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:42:20,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:22,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:42:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:42:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:42:23,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:42:25,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:30,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:42:31,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:36,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:38,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:42:38,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:42:38,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:42:38,609 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:42:40,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:42:41,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:46,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:50,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:42:51,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:42:52,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:42:53,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:42:53,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:42:53,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:42:57,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:00,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:03,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:08,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:43:08,052 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:43:08,053 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:43:08,055 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:43:08,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:43:08,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:43:08,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:43:08,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:43:08,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:09,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:10,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:13,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:18,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:20,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:22,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:43:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:43:23,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:43:23,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:43:24,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:29,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:30,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:34,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:37,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:43:38,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:43:38,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:43:40,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:45,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:47,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:43:51,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:52,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:43:53,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:43:53,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:43:53,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:43:56,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:43:58,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:44:02,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:07,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:08,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:44:08,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:44:08,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:44:08,608 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:44:13,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:18,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:44:19,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:22,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:44:24,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:25,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:44:25,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:44:25,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:44:25,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:44:28,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:44:30,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:35,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:38,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:44:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:44:40,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:44:40,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:44:41,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:46,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:48,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:44:51,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:44:52,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:44:55,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:44:55,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:44:55,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:44:57,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:02,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:45:03,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:08,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:10,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:45:10,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:45:10,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:45:12,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:45:14,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:19,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:22,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:45:22,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:45:24,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:25,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:45:25,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:45:25,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:45:30,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:32,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:45:35,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:40,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:45:40,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:45:40,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:45:41,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:42,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:45:46,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:52,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:45:52,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:45:52,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:45:55,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:45:55,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:45:55,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:45:57,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:02,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:03,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:08,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:10,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:46:10,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:46:10,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:46:12,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:13,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:18,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:22,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:46:23,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:23,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:25,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:46:25,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:46:25,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:46:29,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:30,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:46:30,097 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:46:30,097 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:46:30,100 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:46:31,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:46:31,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:32,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:35,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:40,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:40,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:46:40,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:46:40,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:46:40,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:45,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:50,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:46:50,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:46:52,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:46:55,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:46:55,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:46:55,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:46:56,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:00,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:47:01,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:06,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:10,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:47:10,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:47:10,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:47:10,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:47:12,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:17,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:20,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:47:22,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:22,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:47:25,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:47:25,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:47:25,182 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:47:28,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:30,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:47:33,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:38,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:40,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:47:40,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:47:40,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:47:40,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:47:44,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:50,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:47:50,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:47:52,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:47:55,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:47:55,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:47:55,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:47:55,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:00,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:00,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:48:05,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:10,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:48:10,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:48:10,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:48:10,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:48:11,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:16,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:20,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:48:21,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:22,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:48:25,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:48:25,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:48:25,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:48:27,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:31,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:48:32,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:37,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:40,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:48:40,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:48:40,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:48:41,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:48:43,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:49,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:51,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:48:52,689 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:48:54,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:48:55,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:48:55,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:48:55,186 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:49:00,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:00,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:05,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:10,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:49:10,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:49:10,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:49:10,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:11,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:16,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:20,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:21,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:22,689 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:49:25,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:49:25,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:49:25,184 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:49:27,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:30,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:32,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:37,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:40,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:49:40,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:49:40,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:49:40,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:43,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:48,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:50,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:49:50,151 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:49:50,151 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:49:50,154 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:49:50,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:49:50,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:51,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:49:52,690 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:49:53,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:49:55,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:49:55,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:49:55,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:49:59,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:00,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:50:05,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:10,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:50:10,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:50:10,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:50:10,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:10,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:50:15,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:20,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:22,690 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:50:22,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:50:25,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:50:25,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:50:25,196 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:50:26,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:31,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:32,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:50:36,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:40,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:50:40,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:50:40,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:50:40,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:50:42,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:43,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:50:47,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:52,691 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:50:52,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:50:53,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:50:55,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:50:55,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:50:55,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:50:58,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:03,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:51:03,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:08,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:10,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:51:10,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:51:10,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:51:11,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:51:13,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:18,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:22,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:51:23,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:51:23,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:25,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:51:25,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:51:25,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:51:29,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:31,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:51:35,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:40,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:51:40,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:51:40,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:51:40,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:43,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:51:45,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:50,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:51:51,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:51:52,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:51:55,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:51:55,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:51:55,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:51:56,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:00,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:52:02,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:07,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:10,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:52:10,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:52:10,197 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:52:10,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:52:12,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:17,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:20,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:52:22,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:52:22,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:25,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:52:25,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:52:25,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:52:28,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:30,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:52:34,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:39,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:40,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:52:40,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:52:40,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:52:40,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:52:44,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:49,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:50,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:52:52,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:52:54,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:52:55,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:52:55,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:52:55,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:53:00,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:01,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:05,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:09,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:53:09,434 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:53:09,434 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:53:09,436 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:53:10,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:53:10,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:53:10,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:53:10,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:53:10,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:11,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:12,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:15,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:20,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:21,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:22,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:53:25,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:53:25,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:53:25,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:53:26,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:31,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:31,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:36,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:40,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:53:40,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:53:40,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:53:41,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:42,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:47,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:51,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:53:52,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:53:52,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:53:55,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:53:55,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:53:55,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:53:58,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:01,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:54:04,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:09,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:10,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:54:10,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:54:10,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:54:10,198 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:54:11,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:54:14,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:19,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:21,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:54:22,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:54:24,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:25,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:54:25,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:54:25,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:54:29,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:30,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:54:34,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:40,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:54:40,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:54:40,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:54:40,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:40,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:54:45,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:50,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:54:50,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:54:52,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:54:55,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:54:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:54:55,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:54:55,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:54:56,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:00,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:55:01,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:06,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:10,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:55:10,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:55:10,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:55:10,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:55:10,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:55:12,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:17,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:21,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:55:22,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:22,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:55:25,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:55:25,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:55:25,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:55:25,199 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:55:28,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:31,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:55:34,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:39,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:40,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:55:40,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:55:40,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:55:41,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:55:45,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:50,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:55:51,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:55:52,712 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:55:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:55:55,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:55:55,200 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:55:56,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:01,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:02,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:07,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:10,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:56:10,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:56:10,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:56:11,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:12,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:17,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:21,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:22,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:56:22,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:25,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:56:25,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:56:25,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:56:28,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:29,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 14:56:29,094 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 14:56:29,095 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 14:56:29,098 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 14:56:29,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 14:56:30,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:31,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:34,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:39,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:40,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:56:40,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:56:40,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:56:41,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:44,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:49,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:51,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:56:52,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:56:54,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:56:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:56:55,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:56:55,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:57:00,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:01,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:57:05,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:10,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:57:10,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:57:10,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:57:10,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:57:11,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:11,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:57:16,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:19,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:57:21,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:22,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:57:25,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:57:25,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:57:25,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:57:25,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:57:27,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:28,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:57:32,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:37,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:39,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:57:40,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:57:40,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:57:40,201 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:57:43,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:49,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:57:49,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:52,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:57:54,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:57:55,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:57:55,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:57:55,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:57:59,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:58:00,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:05,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:09,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:58:10,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:58:10,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:58:10,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:58:11,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:16,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:19,325 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:58:21,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:22,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:58:25,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:58:25,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:58:25,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:58:27,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:32,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:37,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:39,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:58:40,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:58:40,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:58:40,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:58:43,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:48,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:49,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:58:52,716 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:58:53,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:58:55,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:58:55,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:58:55,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:58:59,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:01,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:59:04,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:09,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:10,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:59:10,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:59:10,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:59:11,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:59:14,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:20,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:21,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:59:22,717 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:59:25,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:59:25,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:59:25,202 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:59:25,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:31,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:31,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:59:36,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:40,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:59:40,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:59:40,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:59:41,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:41,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:59:46,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:51,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 14:59:52,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 14:59:55,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 14:59:55,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 14:59:55,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 14:59:56,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 14:59:57,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:02,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:03,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:00:03,808 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:00:03,808 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:00:03,811 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:00:04,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:00:06,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:00:07,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:10,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:00:10,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:00:10,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:00:13,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:15,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:00:18,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:22,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:00:24,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:25,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:00:25,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:00:25,203 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:00:25,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:00:30,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:35,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:00:36,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:40,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:00:40,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:00:40,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:00:41,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:45,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:00:46,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:51,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:00:52,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:00:55,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:00:55,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:00:55,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:00:55,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:00:57,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:02,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:05,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:01:07,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:10,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:01:10,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:01:10,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:01:13,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:15,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:01:18,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:22,721 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:01:23,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:25,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:01:25,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:01:25,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:01:25,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:01:29,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:34,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:35,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:01:39,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:40,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:01:40,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:01:40,204 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:01:45,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:45,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:01:50,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:52,722 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:01:55,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:01:55,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:01:55,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:01:55,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:01:55,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:00,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:06,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:06,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:10,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:02:10,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:02:10,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:02:11,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:16,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:17,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:22,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:22,724 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:02:25,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:02:25,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:02:25,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:02:26,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:27,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:32,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:36,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:37,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:40,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:02:40,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:02:40,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:02:43,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:46,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:48,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:52,724 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:02:53,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:02:55,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:02:55,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:02:55,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:02:56,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:02:59,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:04,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:05,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:09,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:10,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:03:10,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:03:10,206 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:03:13,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:15,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:20,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:22,724 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:03:22,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:03:22,820 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:03:22,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:03:22,824 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:03:23,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:03:23,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:24,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:25,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:03:25,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:03:25,205 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:03:25,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:30,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:33,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:35,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:40,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:41,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:03:41,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:03:41,468 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:03:43,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:46,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:51,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:03:52,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:03:53,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:03:56,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:03:56,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:03:56,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:03:56,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:01,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:03,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:04:07,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:11,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:04:11,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:04:11,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:04:12,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:13,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:04:17,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:22,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:04:23,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:23,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:04:26,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:04:26,464 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:04:29,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:34,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:04:35,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:40,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:41,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:04:41,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:04:41,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:04:44,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:04:45,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:50,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:52,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:04:54,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:04:55,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:04:56,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:04:56,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:05:01,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:02,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:05:06,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:11,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:05:11,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:05:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:12,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:05:16,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:21,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:22,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:05:22,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:05:26,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:05:26,465 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:05:27,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:32,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:05:33,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:38,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:41,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:05:41,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:05:41,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:05:41,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:05:43,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:48,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:51,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:05:52,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:05:53,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:05:56,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:05:56,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:05:59,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:01,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:05,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:11,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:11,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:06:11,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:06:11,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:06:11,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:16,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:21,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:22,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:22,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:06:26,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:06:26,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:06:26,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:06:27,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:31,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:33,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:38,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:40,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:06:40,225 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:06:40,226 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:06:40,228 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:06:41,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:06:41,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:06:41,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:06:41,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:06:42,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:43,059 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:43,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:48,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:52,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:06:52,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:06:53,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:06:56,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:06:56,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:06:56,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:06:59,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:02,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:07:05,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:10,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:11,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:07:11,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:07:11,466 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:07:12,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:07:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:21,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:22,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:07:22,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:07:26,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:07:26,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:07:26,477 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:07:27,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:32,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:07:32,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:37,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:41,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:07:41,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:07:41,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:07:42,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:07:43,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:48,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:52,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:07:52,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:07:53,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:07:56,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:07:56,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:07:56,470 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:07:59,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:00,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:04,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:10,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:10,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:11,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:08:11,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:08:15,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:19,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:20,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:21,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:22,730 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:08:26,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:08:26,467 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:08:26,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:31,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:35,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:36,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:41,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:42,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:08:42,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:08:42,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:08:42,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:08:45,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:47,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:52,731 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:08:53,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:08:54,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:08:57,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:08:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:08:57,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:08:58,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:04,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:04,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:09:09,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:12,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:09:12,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:09:12,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:09:12,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:09:14,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:09:15,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:20,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:22,731 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:09:24,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:09:25,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:27,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:09:27,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:09:27,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:09:31,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:34,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:09:37,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:42,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:42,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:09:42,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:09:42,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:09:42,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:09:44,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:09:48,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:52,732 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:09:53,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:09:54,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:09:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:09:57,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:09:59,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:02,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:10:02,764 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:10:02,765 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:10:02,767 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:10:03,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:10:04,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:04,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:05,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:09,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:12,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:10:12,766 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:10:14,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:15,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:20,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:22,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:10:24,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:25,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:27,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:10:27,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:10:31,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:34,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:36,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:41,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:42,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:10:42,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:10:44,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:47,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:52,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:10:52,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:10:54,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:10:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:10:57,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:10:57,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:02,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:04,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:11:08,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:12,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:11:12,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:11:13,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:14,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:11:18,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:22,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:11:24,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:11:24,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:27,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:11:27,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:11:30,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:32,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:11:35,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:40,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:42,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:11:42,767 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:11:44,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:11:45,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:50,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:52,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:11:54,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:11:56,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:11:57,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:11:57,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:12:02,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:04,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:12:07,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:12,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:12,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:12:12,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:12:14,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:12:17,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:22,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:12:23,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:24,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:12:27,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:12:27,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:12:29,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:34,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:34,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:12:39,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:42,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:12:42,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:12:42,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:12:42,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:12:44,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:12:44,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:49,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:52,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:12:52,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:12:55,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:12:57,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:12:57,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:12:57,768 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:13:00,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:02,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:06,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:12,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:12,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:13:12,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:13:12,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:13:12,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:17,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:22,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:13:22,176 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:13:22,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:13:22,180 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:13:22,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:13:22,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:13:22,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:23,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:24,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:27,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:13:27,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:13:27,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:13:28,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:34,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:35,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:39,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:42,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:13:42,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:13:42,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:13:43,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:45,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:50,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:52,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:13:53,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:13:55,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:13:57,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:13:57,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:14:00,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:02,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:14:06,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:11,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:12,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:14:12,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:14:12,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:14:17,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:22,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:14:22,746 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:14:22,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:27,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:27,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:14:27,769 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:14:33,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:36,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:14:38,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:42,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:14:42,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:14:42,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:14:42,773 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:14:43,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:46,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:14:49,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:52,748 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:14:54,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:14:56,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:14:57,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:14:57,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:14:57,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:15:00,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:05,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:06,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:15:10,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:12,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:15:12,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:15:12,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:15:12,775 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:15:15,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:16,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:15:20,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:22,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:15:25,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:26,838 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:15:27,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:15:27,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:15:30,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:35,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:38,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:15:41,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:42,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:15:42,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:15:47,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:49,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:15:52,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:52,750 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:15:57,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:15:57,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:15:57,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:15:57,770 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:16:03,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:07,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:08,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:13,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:13,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:16:13,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:16:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:16:13,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:16:17,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:19,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:22,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:16:24,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:27,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:28,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:16:28,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:16:28,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:16:30,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:36,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:37,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:41,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:43,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:16:43,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:16:43,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:16:43,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:16:44,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:16:44,659 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:16:44,660 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:16:44,662 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:16:45,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:16:45,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:46,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:46,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:51,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:52,753 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:16:56,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:16:57,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:16:58,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:16:58,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:17:03,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:06,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:17:08,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:13,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:17:13,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:17:13,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:17:16,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:17:18,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:22,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:17:23,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:26,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:17:28,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:17:28,898 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:17:29,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:36,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:17:39,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:43,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:17:43,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:17:43,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:17:45,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:46,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:17:50,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:52,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:17:54,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:17:55,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:17:58,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:17:58,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:17:58,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:18:01,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:06,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:07,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:18:11,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:18:13,898 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:18:16,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:17,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:18:21,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:22,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:18:27,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:18:27,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:28,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:18:28,898 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:18:33,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:37,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:18:38,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:43,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:43,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:18:43,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:18:43,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:18:43,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:18:47,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:18:48,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:52,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:18:53,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:55,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:18:58,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:18:58,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:18:58,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:18:58,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:19:04,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:05,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:19:09,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:13,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:19:13,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:19:13,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:19:15,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:15,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:19:20,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:22,760 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:19:24,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:19:25,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:28,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:19:28,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:19:28,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:19:31,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:34,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:19:36,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:41,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:43,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:19:43,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:19:44,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:19:47,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:52,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:52,760 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:19:54,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:19:57,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:19:58,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:19:58,903 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:20:02,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:20:02,008 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:20:02,008 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:20:02,011 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:20:02,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:20:02,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:03,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:03,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:08,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:12,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:13,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:13,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:20:13,900 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:20:19,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:22,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:20:23,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:24,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:29,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:30,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:20:30,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:20:30,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:20:30,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:20:33,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:35,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:40,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:43,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:45,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:20:45,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:20:45,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:20:45,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:20:45,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:50,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:20:52,761 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:20:53,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:20:55,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:00,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:21:00,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:21:00,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:21:01,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:03,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:21:06,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:11,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:13,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:21:15,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:21:15,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:21:15,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:21:17,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:21,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:21:22,763 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:21:22,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:27,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:30,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:21:30,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:21:31,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:21:33,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:38,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:41,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:21:44,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:45,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:21:45,191 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:21:49,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:21:51,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:21:52,764 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:21:54,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:00,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:22:00,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:22:00,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:01,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:22:05,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:10,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:22:10,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:15,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:22:15,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:22:16,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:20,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:22:21,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:22,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:22:26,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:30,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:22:30,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:22:30,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:22:32,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:37,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:41,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:22:42,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:45,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:22:45,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:22:48,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:51,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:22:52,765 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:22:53,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:22:58,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:00,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:23:00,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:23:01,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:04,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:09,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:11,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:14,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:15,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:23:15,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:23:18,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:23:18,601 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:23:18,602 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:23:18,604 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:23:19,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:23:20,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:20,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:21,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:22,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:23:25,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:30,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:23:30,192 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:23:31,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:31,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:36,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:39,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:41,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:46,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:23:46,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:23:46,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:23:47,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:49,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:23:52,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:52,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:23:58,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:23:59,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:24:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:24:01,377 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:24:03,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:09,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:09,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:24:14,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:16,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:24:16,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:24:16,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:24:19,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:19,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:24:22,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:24:24,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:29,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:24:30,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:31,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:24:31,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:24:36,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:39,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:24:41,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:46,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:24:46,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:24:46,380 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:24:46,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:24:47,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:49,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:24:52,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:52,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:24:57,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:24:59,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:25:01,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:25:01,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:25:02,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:08,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:09,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:25:13,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:16,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:25:16,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:25:16,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:25:16,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:25:18,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:19,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:25:22,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:25:24,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:29,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:25:30,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:31,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:25:31,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:25:31,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:25:31,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:25:35,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:39,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:25:41,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:46,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:25:46,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:25:46,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:25:46,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:25:46,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:49,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:25:52,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:52,769 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:25:58,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:25:59,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:01,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:26:01,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:26:01,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:26:03,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:09,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:09,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:14,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:16,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:26:16,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:26:16,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:26:19,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:19,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:22,769 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:26:24,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:29,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:30,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:31,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:26:31,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:26:31,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:26:35,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:36,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:26:36,931 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:26:36,932 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:26:36,934 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:26:37,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:26:37,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:39,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:40,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:45,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:46,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:26:46,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:26:46,646 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:26:49,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:26:51,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:52,770 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:26:57,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:26:57,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:01,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:27:01,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:27:01,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:27:02,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:08,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:10,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:13,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:16,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:27:16,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:27:16,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:27:18,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:20,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:22,770 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:27:24,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:29,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:30,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:31,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:27:31,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:27:31,647 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:27:35,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:39,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:41,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:46,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:46,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:27:46,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:27:46,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:27:46,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:27:49,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:52,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:27:52,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:27:57,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:27:57,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:01,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:28:01,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:28:01,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:28:02,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:08,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:09,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:28:13,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:16,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:28:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:28:16,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:28:16,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:28:18,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:19,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:28:22,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:28:23,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:29,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:29,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:28:31,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:28:31,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:28:31,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:28:34,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:39,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:28:40,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:45,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:46,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:28:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:28:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:28:46,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:28:49,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:28:50,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:52,772 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:28:55,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:28:59,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:29:01,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:01,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:29:01,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:29:01,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:29:06,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:11,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:13,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:29:16,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:29:16,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:29:16,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:29:17,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:22,772 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:29:23,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:24,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:29:28,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:31,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:29:31,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:29:31,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:29:34,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:29:34,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:39,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:44,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:29:45,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:46,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:29:46,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:29:46,649 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:29:50,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:29:52,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:29:54,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:29:56,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:01,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:30:01,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:30:01,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:30:01,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:30:01,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:02,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:30:02,596 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:30:02,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:30:02,599 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:30:03,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:30:04,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:30:07,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:13,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:30:13,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:16,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:30:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:30:16,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:30:18,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:22,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:30:23,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:30:24,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:29,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:31,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:30:31,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:30:31,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:30:31,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:30:33,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:30:35,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:40,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:43,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:30:45,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:30:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:30:46,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:30:51,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:30:52,774 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:30:53,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:30:57,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:01,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:31:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:31:01,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:31:01,654 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:31:03,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:03,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:31:08,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:13,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:31:14,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:31:16,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:31:16,650 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:31:19,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:22,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:31:23,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:31:25,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:30,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:31,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:31:31,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:31:31,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:31:31,653 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:31:34,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:31:36,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:41,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:44,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:31:46,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:31:46,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:31:46,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:31:46,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:51,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:31:52,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:31:54,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:31:57,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:01,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:32:01,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:32:03,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:04,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:32:08,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:14,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:14,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:32:16,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:16,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:32:16,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:32:19,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:22,776 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:32:24,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:32:25,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:30,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:31,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:31,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:32:31,651 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:32:35,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:40,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:44,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:32:46,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:46,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:46,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:32:46,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:32:46,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:32:51,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:32:52,777 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:32:53,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:32:56,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:33:01,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:33:01,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:33:01,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:33:01,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:03,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:06,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:12,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:13,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:33:16,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:33:16,652 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:33:17,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:22,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:33:23,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:23,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:28,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:31,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:33:31,527 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:33:31,528 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:33:31,530 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:33:31,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:33:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:33:31,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:33:32,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:33:33,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:33,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:34,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:38,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:42,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:44,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:46,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:33:46,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:33:46,657 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:33:49,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:33:52,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:33:52,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:33:55,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:00,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:01,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:34:01,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:34:01,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:34:02,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:34:05,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:10,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:14,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:34:16,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:16,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:34:16,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:34:16,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:34:21,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:22,779 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:34:24,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:34:26,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:31,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:34:31,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:34:31,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:34:31,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:32,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:34:36,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:42,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:42,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:34:46,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:34:46,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:34:46,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:34:47,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:52,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:34:52,780 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:34:53,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:34:58,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:01,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:35:01,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:35:01,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:35:02,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:35:04,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:09,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:12,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:35:14,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:16,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:35:16,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:35:16,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:35:19,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:21,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:35:22,780 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:35:25,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:31,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:31,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:35:31,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:35:31,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:35:31,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:35:36,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:41,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:35:42,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:46,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:35:46,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:35:46,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:35:46,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:35:47,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:52,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:35:52,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:35:53,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:35:58,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:01,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:36:01,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:36:01,660 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:36:02,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:36:03,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:08,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:12,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:36:14,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:16,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:36:16,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:36:16,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:36:16,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:36:19,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:22,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:36:22,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:36:25,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:30,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:31,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:36:31,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:36:31,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:36:32,371 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:36:35,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:41,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:42,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:36:46,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:46,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:36:46,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:36:46,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:36:50,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:36:50,743 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:36:50,744 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:36:50,747 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:36:51,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:36:51,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:36:52,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:36:52,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:36:56,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:01,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:37:01,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:37:01,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:37:01,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:02,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:37:06,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:12,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:12,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:37:16,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:37:16,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:37:16,661 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:37:17,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:22,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:37:22,783 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:37:23,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:28,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:31,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:37:31,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:37:31,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:37:32,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:37:33,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:38,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:42,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:37:44,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:46,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:37:46,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:37:46,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:37:49,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:37:52,783 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:37:52,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:37:55,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:00,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:01,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:38:01,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:38:01,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:38:02,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:38:05,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:11,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:12,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:38:16,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:16,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:38:16,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:38:16,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:38:22,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:22,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:38:22,784 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:38:27,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:31,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:38:31,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:38:31,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:38:32,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:38:32,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:37,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:42,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:38:43,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:46,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:38:46,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:38:46,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:38:48,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:52,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:38:52,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:38:54,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:38:59,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:01,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:39:01,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:39:01,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:39:01,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:39:02,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:39:04,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:09,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:12,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:39:15,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:16,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:39:16,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:39:16,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:39:20,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:39:20,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:22,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:39:26,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:30,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:39:31,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:39:31,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:39:31,662 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:39:31,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:36,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:42,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:42,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:39:46,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:39:46,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:39:46,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:39:47,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:52,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:39:52,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:39:53,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:39:58,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:01,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:40:01,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:40:01,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:40:03,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:40:03,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:08,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:10,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:40:10,125 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:40:10,125 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:40:10,128 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:40:11,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:40:11,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:40:13,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:40:14,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:16,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:40:16,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:40:16,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:40:19,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:22,787 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:40:25,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:30,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:31,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:40:31,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:40:31,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:40:35,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:40,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:45,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:46,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:40:46,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:40:46,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:40:51,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:40:52,787 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:40:57,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:01,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:41:01,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:41:01,663 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:41:02,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:07,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:12,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:16,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:41:16,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:41:16,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:41:17,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:22,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:41:23,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:28,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:31,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:41:31,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:41:31,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:41:33,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:38,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:43,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:46,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:41:46,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:41:46,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:41:48,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:52,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:41:54,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:41:59,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:01,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:42:01,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:42:01,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:42:04,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:09,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:14,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:16,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:42:16,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:42:16,664 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:42:19,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:22,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:42:25,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:30,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:31,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:42:31,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:42:31,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:42:35,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:40,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:45,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:46,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:42:46,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:42:46,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:42:51,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:42:52,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:42:57,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:01,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:43:01,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:43:01,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:43:02,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:07,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:12,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:16,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:43:16,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:43:16,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:43:17,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:22,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:43:23,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:28,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:31,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:43:31,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:43:31,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:43:33,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:38,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:43,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:46,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:43:46,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:43:46,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:43:49,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:43:52,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:43:55,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:00,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:01,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:44:01,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:44:01,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:44:05,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:10,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:15,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:16,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:44:16,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:44:16,665 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:44:20,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:22,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:44:26,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:31,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:44:31,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:44:31,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:44:31,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:36,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:41,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:46,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:44:46,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:44:46,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:44:46,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:51,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:44:52,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:44:57,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:01,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:45:01,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:45:01,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:45:02,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:07,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:12,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:16,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:45:16,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:45:16,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:45:18,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:22,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:45:24,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:29,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:31,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:45:31,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:45:31,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:45:34,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:39,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:44,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:46,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:45:46,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:45:46,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:45:50,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:45:52,793 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:45:56,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:01,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:46:01,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:46:01,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:46:01,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:06,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:11,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:16,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:46:16,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:46:16,666 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:46:16,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:21,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:22,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:46:27,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:31,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:46:31,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:46:31,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:46:32,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:37,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:42,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:46,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:46:46,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:46:46,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:46:48,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:52,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:46:54,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:46:59,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:01,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:47:01,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:47:01,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:47:04,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:09,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:14,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:16,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:47:16,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:47:16,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:47:20,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:22,794 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:47:26,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:31,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:47:31,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:47:31,667 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:47:31,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:36,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:41,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:46,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:47:46,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:47:46,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:47:47,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:52,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:47:53,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:47:58,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:01,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:48:01,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:48:01,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:48:03,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:08,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:16,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:48:16,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:48:16,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:48:19,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:22,795 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:48:25,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:30,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:31,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:48:31,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:48:31,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:48:35,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:40,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:45,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:46,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:48:46,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:48:46,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:48:50,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:48:52,796 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:48:56,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:01,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:49:01,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:49:01,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:49:01,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:06,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:11,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:16,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:49:16,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:49:16,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:49:17,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:22,797 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:49:23,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:28,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:31,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:49:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:49:31,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:49:33,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:38,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:43,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:46,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:49:46,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:49:46,668 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:49:48,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:52,797 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:49:54,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:49:59,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:01,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:50:01,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:50:01,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:50:04,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:09,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:14,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:16,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:50:16,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:50:16,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:50:20,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:22,797 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:50:26,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:31,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:50:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:50:31,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:50:31,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:36,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:41,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:46,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:50:46,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:50:46,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:50:47,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:52,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:50:53,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:50:58,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:01,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:51:01,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:51:01,669 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:51:03,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:06,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:51:06,311 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:51:06,312 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:51:06,317 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:51:06,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:51:07,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:51:09,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:14,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:16,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:51:16,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:51:16,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:51:19,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:22,798 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:51:25,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:30,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:51:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:51:31,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:51:35,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:39,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:51:41,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:46,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:46,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:51:47,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:51:47,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:51:47,411 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:51:51,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:51:52,799 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:51:56,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:01,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:02,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:52:02,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:52:02,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:52:02,164 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:52:07,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:12,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:16,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:52:16,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:52:18,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:22,800 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:52:23,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:27,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:52:29,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:34,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:39,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:52:39,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:39,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:52:39,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: stop_status -2023-11-20 15:52:39,619 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: stop_status -2023-11-20 15:52:42,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:52:44,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:49,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:52:52,800 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:52:55,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:00,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:01,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:53:05,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:10,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:15,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:20,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:21,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:53:22,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:53:25,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:30,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:35,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:40,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:42,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:53:43,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:53:45,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:47,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:53:50,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:52,807 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:53:53,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:53:56,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:53:58,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:01,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:03,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:06,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:08,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:11,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:13,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:16,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:18,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:21,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:22,808 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:54:23,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:27,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:28,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:32,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:33,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:37,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:38,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:42,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:43,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:47,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:54:47,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:48,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:52,808 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:54:53,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:54:53,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:54:58,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:03,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:04,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:04,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:07,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:55:09,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:09,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:14,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:14,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:18,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:55:19,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:19,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:22,809 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:55:24,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:25,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:28,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:55:29,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:30,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:34,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:36,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:36,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:40,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:55:40,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:41,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:45,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:50,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:55:50,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:51,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:55:52,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:55:56,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:55:56,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:00,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:56:01,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:02,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:07,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:07,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:07,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:10,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:56:12,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:13,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:18,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:19,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:19,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:56:22,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:56:23,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:24,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:29,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:29,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:56:30,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:34,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:35,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:40,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:41,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:41,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:56:45,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:46,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:50,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:51,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:56:52,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:56:52,811 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:56:56,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:56:57,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:01,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:57:02,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:03,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:08,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:08,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:08,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:11,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:57:13,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:13,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:18,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:18,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:21,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:57:22,811 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:57:23,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:23,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:28,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:28,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:32,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:57:33,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:34,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:39,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:39,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:42,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:57:44,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:44,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:49,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:49,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:52,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:57:52,812 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:57:54,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:57:54,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:57:59,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:00,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:02,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:58:05,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:05,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:10,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:11,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:12,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:58:15,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:16,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:20,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:20,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 15:58:20,489 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 15:58:20,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 15:58:20,493 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 15:58:21,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:21,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 15:58:22,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:58:22,812 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:58:25,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:26,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:30,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:32,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:32,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:58:36,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:37,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:41,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:42,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:43,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:58:46,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:47,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:51,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:52,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:52,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:58:52,813 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:58:53,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:58:56,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:58:57,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:01,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:02,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:06,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:07,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 15:59:07,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:12,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:12,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:17,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:17,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:22,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:22,813 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:59:22,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:27,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:27,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:32,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:32,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:37,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:37,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:42,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:42,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:47,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:47,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:52,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 15:59:52,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:53,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 15:59:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 15:59:58,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:02,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:00:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:04,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:08,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:08,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:09,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:12,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:00:13,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:14,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:19,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:19,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:22,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:00:22,818 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:00:24,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:24,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:29,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:29,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:32,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:00:34,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:35,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:40,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:40,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:40,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:44,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:00:45,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:45,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:50,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:50,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:50,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:00:52,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:00:54,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:00:55,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:00:56,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:00,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:02,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:04,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:01:06,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:07,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:11,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:12,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:14,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:01:17,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:17,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:22,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:22,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:01:23,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:23,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:24,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:01:27,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:28,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:32,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:33,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:36,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:01:37,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:38,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:42,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:44,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:45,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:01:48,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:49,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:52,820 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:01:53,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:01:54,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:01:55,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:01:59,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:00,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:04,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:02:05,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:05,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:10,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:12,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:12,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:14,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:02:15,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:17,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:20,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:22,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:02:23,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:24,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:02:26,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:28,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:31,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:33,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:02:33,321 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:02:33,322 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:02:33,325 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:02:34,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:34,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:02:34,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:02:36,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:39,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:41,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:44,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:02:44,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:47,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:49,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:52,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:02:52,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:02:54,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:02:55,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:02:58,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:02,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:03,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:04,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:03:07,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:08,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:13,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:14,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:03:14,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:18,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:19,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:22,822 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:03:23,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:24,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:03:25,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:28,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:30,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:34,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:36,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:36,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:03:39,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:41,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:44,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:46,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:03:46,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:49,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:51,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:52,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:03:54,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:54,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:03:56,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:03:57,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:03:59,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:02,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:04,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:06,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:04:07,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:09,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:12,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:14,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:16,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:04:18,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:19,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:22,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:04:23,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:24,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:24,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:26,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:04:29,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:29,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:34,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:34,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:38,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:04:39,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:40,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:44,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:46,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:48,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:04:50,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:51,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:52,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:04:55,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:04:56,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:04:58,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:05:01,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:01,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:06,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:06,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:08,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:05:11,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:11,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:17,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:17,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:18,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:05:22,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:22,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:22,826 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:05:27,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:27,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:28,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:05:32,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:32,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:37,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:38,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:38,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:05:42,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:43,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:48,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:48,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:48,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:05:52,826 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:05:53,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:53,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:05:58,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:05:58,090 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:05:58,090 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:05:58,093 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:05:58,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:05:58,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:05:59,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:00,784 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:06:03,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:04,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:09,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:09,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:10,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:06:14,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:14,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:20,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:20,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:20,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:06:22,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:06:25,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:25,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:30,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:30,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:30,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:06:31,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:35,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:36,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:41,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:06:41,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:42,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:46,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:47,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:51,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:06:52,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:06:52,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:06:52,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:57,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:06:58,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:00,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:07:03,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:03,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:08,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:08,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:10,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:07:13,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:14,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:19,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:20,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:20,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:07:22,828 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:07:24,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:25,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:29,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:30,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:07:31,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:34,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:36,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:40,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:40,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:07:41,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:45,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:46,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:50,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:50,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:07:51,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:07:52,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:07:55,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:07:57,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:00,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:08:00,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:00,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:02,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:05,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:07,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:10,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:08:11,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:12,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:16,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:17,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:20,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:08:21,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:22,830 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:08:23,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:26,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:28,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:30,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:08:31,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:33,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:36,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:38,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:41,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:08:42,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:48,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:48,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:51,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:08:52,831 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:08:53,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:54,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:08:58,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:08:58,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:00,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:01,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:09:04,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:05,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:10,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:11,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:09:11,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:15,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:16,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:19,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:09:19,066 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:09:19,066 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:09:19,068 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:09:19,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:09:20,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:21,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:09:22,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:22,831 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:09:25,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:27,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:31,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:31,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:09:33,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:36,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:38,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:40,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:09:41,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:43,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:46,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:48,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:50,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:09:52,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:52,832 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:09:53,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:09:58,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:09:58,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:00,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:10:03,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:04,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:08,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:09,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:10,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:10:13,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:14,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:19,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:19,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:20,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:10:22,833 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:10:24,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:24,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:29,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:30,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:30,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:10:34,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:35,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:39,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:40,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:10:41,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:44,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:46,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:50,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:51,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:10:51,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:10:52,833 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:10:55,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:10:57,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:00,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:01,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:11:03,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:06,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:08,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:11,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:11:12,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:14,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:18,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:19,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:21,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:11:22,834 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:11:23,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:24,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:28,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:30,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:31,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:11:33,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:35,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:39,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:41,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:11:41,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:44,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:46,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:50,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:51,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:11:52,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:11:52,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:11:55,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:11:57,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:00,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:01,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:12:03,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:05,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:08,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:10,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:11,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:12:14,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:15,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:19,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:20,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:12:21,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:22,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:12:24,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:27,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:30,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:30,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:12:32,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:35,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:38,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:39,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:12:39,304 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:12:39,305 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:12:39,307 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:12:39,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:12:40,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:40,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:12:43,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:45,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:49,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:50,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:12:51,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:52,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:12:54,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:12:56,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:12:59,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:00,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:13:02,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:05,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:07,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:10,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:11,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:13:12,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:15,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:17,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:21,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:13:21,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:22,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:13:22,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:26,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:27,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:31,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:13:31,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:33,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:37,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:38,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:41,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:13:42,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:44,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:48,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:50,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:51,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:13:52,837 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:13:53,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:13:55,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:13:59,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:01,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:01,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:14:04,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:06,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:10,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:11,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:14:11,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:15,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:16,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:21,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:21,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:14:22,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:22,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:14:26,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:27,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:31,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:14:31,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:33,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:37,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:38,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:42,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:43,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:45,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:14:47,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:48,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:52,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:14:53,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:54,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:14:55,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:14:58,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:14:59,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:04,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:04,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:07,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:15:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:09,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:14,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:15,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:15,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:16,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:15:20,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:20,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:22,841 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:15:25,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:26,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:27,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:15:30,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:31,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:35,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:36,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:37,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:15:40,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:41,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:45,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:15:46,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:47,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:51,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:52,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:15:52,841 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:15:55,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:15:57,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:15:57,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:02,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:03,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:03,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:04,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:16:04,948 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:16:04,949 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:16:04,952 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:16:05,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:16:07,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:16:07,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:08,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:12,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:14,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:14,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:17,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:16:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:19,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:22,843 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:16:24,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:24,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:27,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:16:29,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:29,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:35,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:16:35,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:35,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:40,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:40,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:45,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:16:45,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:46,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:50,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:51,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:16:52,844 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:16:55,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:16:55,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:16:56,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:00,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:01,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:05,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:05,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:17:07,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:10,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:12,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:15,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:17:16,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:18,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:21,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:22,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:17:23,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:25,875 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:17:26,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:29,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:31,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:34,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:35,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:17:37,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:39,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:43,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:44,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:45,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:17:48,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:49,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:52,846 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:17:54,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:17:55,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:17:55,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:17:59,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:00,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:05,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:05,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:18:06,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:10,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:11,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:15,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:18:15,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:17,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:20,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:22,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:22,861 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:18:25,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:18:26,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:27,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:31,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:32,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:35,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:18:37,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:37,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:42,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:43,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:45,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:18:48,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:48,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:52,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:18:53,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:18:53,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:58,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:18:59,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:03,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:04,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:08,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:13,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:14,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:17,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:19:19,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:19,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:22,848 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:19:24,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:25,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:27,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:19:30,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:30,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:35,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:36,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:40,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:19:40,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:41,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:45,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:46,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:47,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:48,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:19:48,289 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:19:48,290 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:19:48,293 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:19:49,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:19:49,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:19:50,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:19:51,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:51,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:19:52,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:19:56,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:19:57,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:01,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:02,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:06,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:07,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:11,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:12,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:16,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:17,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:22,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:22,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:20:23,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:23,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:20:27,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:30,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:30,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:33,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:33,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:20:35,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:38,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:40,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:43,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:43,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:20:45,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:48,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:51,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:52,850 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:20:53,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:20:53,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:20:56,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:20:58,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:02,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:02,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:03,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:21:04,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:07,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:09,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:12,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:15,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:16,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:21:17,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:20,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:22,851 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:21:22,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:26,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:21:26,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:27,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:31,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:33,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:36,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:21:36,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:38,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:41,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:44,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:46,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:21:47,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:49,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:52,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:52,851 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:21:54,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:21:56,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:21:58,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:21:59,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:03,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:05,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:22:05,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:08,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:10,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:13,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:15,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:22:16,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:18,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:22,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:22,853 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:22:24,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:25,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:22:27,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:29,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:33,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:33,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:35,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:35,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:22:38,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:40,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:43,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:45,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:22:46,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:48,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:51,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:52,853 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:22:53,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:22:54,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:22:56,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:22:59,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:01,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:03,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:05,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:07,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:11,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:12,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:15,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:16,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:18,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:21,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:21,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:22,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:23:23,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:26,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:26,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:28,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:32,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:32,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:33,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:23:33,154 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:23:33,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:23:33,156 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:23:34,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:23:34,097 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:34,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:36,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:37,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:39,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:43,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:44,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:44,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:48,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:49,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:52,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:23:54,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:23:54,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:23:55,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:23:59,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:00,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:04,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:24:05,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:06,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:11,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:11,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:14,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:24:16,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:17,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:22,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:22,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:22,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:24:24,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:24:27,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:27,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:32,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:32,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:36,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:24:37,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:38,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:43,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:43,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:44,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:24:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:48,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:52,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:24:53,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:24:53,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:24:54,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:24:58,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:01,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:03,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:25:04,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:06,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:09,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:11,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:13,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:25:15,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:16,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:20,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:21,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:21,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:22,856 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:25:23,844 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:25:25,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:26,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:30,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:32,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:32,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:33,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:25:35,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:37,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:40,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:42,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:44,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:25:46,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:47,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:52,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:52,856 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:25:53,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:25:54,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:25:57,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:25:59,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:02,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:26:03,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:04,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:08,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:10,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:12,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:26:13,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:15,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:18,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:21,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:21,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:22,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:26:22,857 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:26:24,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:26,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:29,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:32,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:32,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:32,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:26:34,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:37,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:39,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:42,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:26:42,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:44,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:47,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:49,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:51,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:26:51,235 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:26:51,236 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:26:51,239 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:26:51,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:26:52,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:26:52,858 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:26:52,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:26:55,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:26:57,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:00,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:02,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:27:02,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:06,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:07,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:12,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:12,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:27:13,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:17,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:19,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:19,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:22,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:22,761 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:27:22,858 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:27:24,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:27,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:30,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:31,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:27:33,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:35,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:38,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:41,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:41,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:27:44,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:46,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:49,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:51,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:51,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:51,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:27:52,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:27:54,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:27:56,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:27:59,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:02,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:28:02,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:05,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:07,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:10,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:12,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:28:12,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:16,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:17,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:21,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:22,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:28:22,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:28:23,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:26,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:29,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:31,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:32,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:28:34,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:36,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:40,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:42,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:28:42,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:45,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:47,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:51,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:51,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:52,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:28:52,861 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:28:53,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:28:56,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:28:58,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:01,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:02,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:29:04,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:06,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:09,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:13,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:14,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:16,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:29:18,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:19,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:22,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:29:24,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:24,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:26,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:29:29,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:30,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:34,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:34,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:35,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:38,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:29:39,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:40,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:44,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:45,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:48,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:29:50,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:51,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:29:52,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:29:56,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:29:56,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:00,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:30:01,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:02,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:06,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:07,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:11,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:12,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:14,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:30:16,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:18,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:22,864 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:30:22,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:23,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:24,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:30:24,111 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:30:24,111 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:30:24,114 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:30:25,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:30:26,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:30:27,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:29,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:33,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:33,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:34,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:36,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:30:38,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:39,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:43,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:44,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:46,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:30:48,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:49,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:52,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:30:54,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:30:54,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:30:56,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:30:59,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:00,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:04,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:04,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:04,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:05,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:06,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:31:09,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:10,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:14,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:15,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:18,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:31:19,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:20,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:22,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:31:25,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:25,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:28,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:31:30,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:31,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:35,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:35,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:35,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:36,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:38,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:31:40,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:41,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:45,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:46,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:48,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:31:50,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:52,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:31:52,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:31:55,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:31:57,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:00,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:32:01,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:03,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:07,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:09,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:10,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:32:12,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:14,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:17,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:20,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:20,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:32:22,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:22,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:32:25,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:27,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:31,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:31,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:32:32,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:36,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:37,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:40,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:32:41,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:42,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:46,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:49,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:51,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:52,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:32:52,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:32:54,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:32:56,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:32:59,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:02,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:33:02,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:04,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:07,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:10,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:10,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:12,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:33:13,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:15,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:18,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:20,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:22,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:33:23,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:24,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:33:25,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:28,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:30,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:34,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:34,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:33:35,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:39,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:41,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:44,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:33:44,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:46,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:49,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:52,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:52,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:52,869 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:33:53,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:33:53,976 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:33:53,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:33:53,981 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:33:54,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:33:54,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:33:56,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:33:57,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:33:59,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:02,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:05,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:06,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:34:07,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:10,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:12,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:15,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:16,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:34:18,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:20,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:22,871 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:34:23,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:23,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:25,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:26,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:34:28,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:30,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:33,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:35,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:37,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:34:38,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:40,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:44,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:46,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:47,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:34:49,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:51,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:52,887 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:34:54,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:54,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:34:56,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:34:57,149 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:34:59,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:01,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:05,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:06,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:07,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:35:10,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:11,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:15,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:17,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:17,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:35:20,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:22,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:22,874 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:35:26,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:27,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:35:28,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:31,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:33,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:36,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:35:37,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:39,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:42,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:44,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:46,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:35:47,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:50,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:52,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:35:52,875 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:35:55,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:35:56,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:35:58,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:00,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:04,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:05,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:08,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:36:09,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:11,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:15,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:17,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:18,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:36:20,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:22,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:22,875 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:36:25,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:27,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:28,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:36:30,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:32,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:36,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:36,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:37,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:40,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:36:41,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:43,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:47,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:48,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:36:49,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:52,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:52,876 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:36:54,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:36:58,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:36:59,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:37:00,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:03,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:05,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:08,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:09,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:37:11,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:13,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:16,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:18,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:37:18,557 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:37:18,558 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:37:18,562 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:37:19,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:37:19,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:21,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:37:21,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:22,876 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:37:24,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:26,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:30,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:31,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:37:32,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:36,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:36,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:37,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:39,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:37:41,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:43,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:47,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:48,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:49,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:37:52,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:52,878 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:37:53,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:58,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:37:58,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:37:59,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:38:03,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:03,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:08,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:09,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:38:09,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:14,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:14,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:19,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:20,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:20,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:38:22,888 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:38:24,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:25,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:30,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:30,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:38:30,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:35,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:35,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:40,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:40,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:40,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:38:40,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:45,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:45,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:50,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:38:51,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:51,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:52,889 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:38:56,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:38:57,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:38:57,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:00,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:39:02,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:02,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:07,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:07,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:11,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:39:12,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:13,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:17,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:18,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:21,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:39:22,889 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:39:22,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:23,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:28,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:28,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:31,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:39:33,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:34,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:38,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:38,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:38,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:39,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:41,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:39:43,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:44,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:48,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:50,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:51,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:39:52,890 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:39:53,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:39:55,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:39:59,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:01,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:01,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:40:04,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:06,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:09,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:11,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:40:12,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:14,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:17,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:20,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:21,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:40:22,890 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:40:22,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:25,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:27,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:31,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:40:31,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:33,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:36,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:38,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:40,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:40:40,534 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:40:40,535 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:40:40,537 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:40:40,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:40:41,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:43,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:43,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:40:47,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:48,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:52,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:52,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:40:53,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:40:53,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:40:57,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:40:58,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:02,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:41:02,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:04,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:07,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:09,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:12,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:41:13,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:13,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:15,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:18,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:20,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:22,892 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:41:23,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:41:24,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:25,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:29,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:30,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:33,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:41:34,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:36,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:39,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:41,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:43,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:41:44,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:47,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:50,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:52,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:41:52,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:41:53,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:41:55,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:41:57,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:00,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:03,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:03,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:42:05,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:08,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:10,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:10,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:10,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:13,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:42:13,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:15,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:18,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:21,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:22,893 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:42:24,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:25,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:42:26,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:29,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:32,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:35,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:42:35,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:37,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:40,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:43,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:45,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:48,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:51,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:52,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:42:53,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:42:53,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:42:56,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:42:59,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:01,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:04,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:06,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:11,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:13,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:13,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:13,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:17,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:43:17,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:18,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:22,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:22,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:43:24,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:27,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:43:27,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:29,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:32,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:35,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:37,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:43:38,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:40,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:43,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:46,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:46,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:49,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:49,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:43:51,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:52,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:43:54,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:43:56,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:43:59,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:43:59,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:01,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:05,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:06,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:09,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:44:10,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:11,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:15,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:17,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:17,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:19,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:44:20,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:22,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:22,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:44:25,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:27,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:44:27,277 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:44:27,278 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:44:27,280 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:44:27,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:44:27,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:29,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:44:31,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:32,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:36,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:38,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:39,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:44:41,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:43,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:46,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:48,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:49,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:44:52,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:52,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:44:53,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:57,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:44:59,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:44:59,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:45:03,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:04,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:08,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:09,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:09,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:45:13,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:14,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:18,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:19,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:21,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:45:22,907 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:45:23,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:24,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:28,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:29,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:32,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:45:34,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:35,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:39,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:40,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:42,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:45:44,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:45,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:49,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:45:50,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:52,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:45:55,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:45:55,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:00,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:00,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:05,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:07,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:09,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:46:11,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:12,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:16,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:17,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:17,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:19,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:46:21,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:22,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:22,909 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:46:26,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:27,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:29,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:46:32,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:32,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:37,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:37,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:39,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:46:42,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:43,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:48,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:48,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:49,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:46:52,909 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:46:53,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:53,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:46:58,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:46:59,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:46:59,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:03,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:04,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:09,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:09,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:11,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:47:14,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:14,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:19,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:19,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:21,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:47:22,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:47:24,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:25,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:30,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:30,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:31,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:31,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:47:35,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:36,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:41,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:41,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:47:42,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:46,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:47,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:52,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:47:52,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:52,910 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:47:52,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:47:57,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:47:57,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:02,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:48:02,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:02,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:03,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:07,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:08,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:10,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:48:10,782 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:48:10,783 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:48:10,786 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:48:11,178 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:48:12,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:48:13,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:14,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:19,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:19,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:19,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:22,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:48:22,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:48:24,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:24,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:29,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:29,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:29,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:32,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:48:35,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:35,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:40,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:41,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:42,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:48:46,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:46,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:51,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:48:51,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:52,911 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:48:53,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:48:56,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:48:57,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:01,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:02,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:06,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:07,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:11,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:12,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:16,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:17,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:22,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:22,912 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:49:22,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:23,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:49:27,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:27,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:32,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:33,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:34,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:49:37,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:38,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:42,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:44,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:46,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:49:47,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:49,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:52,914 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:49:53,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:54,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:49:56,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:49:58,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:49:59,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:04,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:04,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:05,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:06,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:50:09,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:10,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:15,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:16,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:50:16,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:20,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:21,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:22,916 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:50:25,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:26,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:50:26,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:30,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:31,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:35,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:35,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:36,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:50:36,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:40,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:41,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:45,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:50:46,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:47,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:51,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:52,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:50:52,916 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:50:55,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:50:56,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:50:57,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:01,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:02,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:05,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:51:06,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:08,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:12,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:13,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:15,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:51:17,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:18,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:22,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:51:23,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:23,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:25,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:51:28,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:28,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:33,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:33,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:36,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:51:38,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:39,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:44,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:44,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:48,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:51:49,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:50,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:52,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:51:54,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:51:55,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:51:56,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:51:56,346 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:51:56,347 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:51:56,354 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:51:57,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:51:58,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:51:59,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:01,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:05,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:05,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:07,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:08,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:52:10,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:12,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:16,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:17,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:18,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:52:21,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:22,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:22,919 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:52:26,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:27,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:28,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:52:31,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:32,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:37,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:38,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:38,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:52:42,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:43,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:47,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:48,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:52:49,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:52,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:52:53,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:52:54,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:52:58,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:00,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:53:00,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:03,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:05,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:08,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:53:09,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:11,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:14,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:16,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:17,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:53:19,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:22,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:22,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:53:26,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:27,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:27,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:53:31,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:32,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:36,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:36,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:37,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:39,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:53:41,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:42,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:47,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:48,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:50,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:53:52,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:52,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:53:53,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:53:58,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:53:59,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:00,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:54:03,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:04,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:08,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:54:08,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:10,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:13,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:15,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:18,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:20,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:54:20,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:22,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:54:24,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:25,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:29,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:30,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:54:31,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:34,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:36,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:39,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:40,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:54:41,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:44,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:46,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:49,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:50,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:54:52,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:54:52,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:54:55,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:54:57,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:00,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:01,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:03,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:07,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:08,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:10,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:12,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:14,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:18,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:19,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:20,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:55:20,173 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:55:20,174 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:55:20,176 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:55:20,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:55:22,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:22,927 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:55:23,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:24,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:29,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:29,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:32,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:34,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:35,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:40,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:41,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:42,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:45,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:46,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:47,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:48,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:55:50,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:51,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:55:52,927 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:55:56,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:55:57,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:00,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:56:01,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:03,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:06,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:06,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:08,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:10,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:56:11,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:13,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:17,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:19,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:20,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:56:22,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:22,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:56:24,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:27,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:30,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:56:30,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:32,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:35,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:38,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:40,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:56:41,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:43,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:46,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:48,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:50,296 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:56:52,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:52,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:56:53,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:56:57,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:56:59,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:00,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:57:03,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:04,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:08,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:10,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:10,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:57:14,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:15,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:19,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:20,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:57:20,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:22,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:57:24,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:25,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:30,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:57:30,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:31,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:35,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:36,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:41,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:42,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:42,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:57:46,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:47,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:51,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:52,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:57:52,930 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:57:53,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:53,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:57:56,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:57:59,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:02,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:02,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:04,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:07,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:10,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:12,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:13,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:15,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:18,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:20,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:20,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:22,930 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:58:23,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:25,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:28,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:31,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:33,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:34,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:36,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:39,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:40,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 16:58:40,485 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 16:58:40,485 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 16:58:40,487 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 16:58:41,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 16:58:42,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:42,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:43,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:44,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:47,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:49,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:52,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:58:52,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:52,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:58:54,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:58:57,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:58:59,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:02,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:02,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:05,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:07,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:10,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:12,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:12,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:15,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:17,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:20,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:22,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:22,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:22,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:22,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:59:26,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:27,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:31,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:32,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:32,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:37,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:37,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:42,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:42,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:43,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:47,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:48,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:52,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:52,943 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 16:59:52,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 16:59:57,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 16:59:58,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 16:59:59,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:00,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:00:03,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:05,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:08,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:10,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:12,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:00:13,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:15,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:18,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:21,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:22,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:00:22,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:00:23,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:26,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:28,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:32,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:32,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:00:33,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:37,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:38,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:42,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:42,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:00:44,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:47,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:49,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:52,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:00:53,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:00:53,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:54,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:00:58,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:00:59,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:03,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:01:03,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:05,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:09,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:09,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:10,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:13,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:01:14,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:15,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:19,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:20,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:22,934 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:01:25,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:25,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:29,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:01:30,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:31,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:35,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:37,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:38,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:01:40,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:42,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:46,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:48,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:48,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:01:51,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:52,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:01:53,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:01:56,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:01:56,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:01:59,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:01,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:04,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:06,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:02:06,180 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:02:06,181 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:02:06,186 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:02:06,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:02:06,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:06,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:08,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:09,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:12,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:14,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:17,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:18,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:19,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:22,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:22,936 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:02:24,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:28,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:28,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:30,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:33,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:35,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:38,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:38,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:40,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:43,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:45,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:46,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:49,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:51,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:52,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:02:54,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:54,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:02:56,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:02:57,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:02:59,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:02,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:05,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:05,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:07,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:09,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:03:10,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:12,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:16,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:18,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:19,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:03:21,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:22,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:03:23,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:27,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:29,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:29,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:03:32,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:34,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:37,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:39,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:03:40,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:43,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:45,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:48,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:49,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:03:50,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:52,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:03:53,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:03:56,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:03:59,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:03:59,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:01,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:04,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:06,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:09,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:04:10,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:12,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:16,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:18,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:19,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:04:21,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:22,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:04:23,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:27,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:28,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:04:29,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:32,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:34,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:37,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:38,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:04:39,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:42,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:44,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:48,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:48,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:04:50,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:52,940 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:04:53,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:55,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:04:58,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:04:58,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:05:01,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:03,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:06,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:08,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:05:09,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:12,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:14,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:18,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:18,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:05:19,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:22,940 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:05:23,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:24,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:27,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:05:27,457 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:05:27,458 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:05:27,467 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:05:28,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:05:29,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:05:29,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:30,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:34,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:36,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:39,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:05:40,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:41,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:45,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:47,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:49,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:05:51,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:52,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:52,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:05:56,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:05:57,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:05:59,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:06:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:02,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:07,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:08,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:09,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:06:13,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:13,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:18,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:19,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:06:19,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:22,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:06:23,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:24,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:29,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:29,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:06:30,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:34,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:35,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:39,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:06:39,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:41,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:44,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:47,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:49,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:06:50,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:52,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:06:52,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:06:55,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:06:57,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:01,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:07:01,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:02,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:06,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:07,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:07,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:07,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:11,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:07:11,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:12,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:16,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:18,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:20,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:07:22,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:22,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:07:23,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:27,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:28,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:30,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:07:33,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:33,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:38,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:38,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:38,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:38,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:40,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:07:43,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:43,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:48,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:49,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:51,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:07:52,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:07:53,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:07:54,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:07:58,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:00,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:01,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:08:04,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:05,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:09,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:10,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:11,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:08:14,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:15,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:19,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:20,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:21,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:08:22,943 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:08:24,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:25,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:29,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:31,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:33,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:08:35,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:36,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:41,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:42,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:43,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:08:46,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:47,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:50,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:08:50,559 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:08:50,560 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:08:50,561 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:08:51,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:08:51,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:52,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:08:52,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:08:53,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:08:56,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:08:57,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:02,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:03,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:03,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:09:07,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:09,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:11,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:09:13,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:14,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:18,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:19,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:21,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:09:22,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:09:23,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:24,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:28,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:30,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:31,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:09:34,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:35,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:39,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:40,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:43,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:09:44,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:45,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:49,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:50,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:52,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:09:53,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:09:54,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:09:55,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:09:59,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:01,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:02,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:10:05,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:06,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:11,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:10:11,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:12,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:12,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:16,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:17,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:21,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:10:22,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:22,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:22,946 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:10:27,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:27,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:32,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:33,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:33,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:10:38,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:39,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:39,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:39,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:43,241 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:10:43,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:44,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:48,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:49,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:52,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:10:53,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:10:53,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:10:54,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:10:58,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:00,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:03,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:11:04,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:05,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:09,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:11,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:13,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:11:15,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:16,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:20,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:22,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:22,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:11:23,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:11:25,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:27,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:30,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:32,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:33,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:11:36,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:37,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:42,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:43,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:11:43,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:47,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:49,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:51,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:11:52,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:11:52,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:11:54,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:11:57,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:00,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:01,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:12:03,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:05,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:08,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:10,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:12:10,322 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:12:10,323 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:12:10,327 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:12:10,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:10,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:12:11,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:12:14,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:15,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:19,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:21,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:22,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:12:23,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:12:24,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:26,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:29,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:32,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:12:32,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:35,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:37,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:40,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:42,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:44,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:12:46,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:47,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:51,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:52,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:12:53,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:12:53,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:56,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:12:59,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:12:59,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:02,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:03,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:13:04,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:07,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:10,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:13,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:13:13,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:15,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:18,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:21,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:22,949 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:13:23,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:13:23,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:26,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:28,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:31,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:33,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:13:34,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:36,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:39,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:42,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:43,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:13:45,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:50,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:52,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:13:52,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:13:53,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:13:55,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:13:57,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:00,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:02,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:03,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:14:06,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:07,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:11,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:13,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:14:13,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:17,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:18,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:22,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:22,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:14:23,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:14:24,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:27,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:31,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:33,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:33,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:14:36,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:38,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:41,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:43,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:14:44,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:46,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:49,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:51,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:52,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:14:54,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:14:54,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:14:56,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:14:59,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:02,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:04,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:05,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:07,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:10,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:13,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:13,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:14,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:16,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:18,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:21,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:22,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:15:23,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:24,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:26,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:28,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:31,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:15:31,907 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:15:31,908 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:15:31,911 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:15:32,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:15:32,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:33,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:34,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:34,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:37,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:40,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:43,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:44,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:45,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:48,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:51,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:52,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:15:53,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:15:53,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:15:56,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:15:58,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:01,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:03,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:16:03,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:06,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:08,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:11,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:13,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:17,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:19,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:19,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:16:22,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:22,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:16:24,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:28,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:29,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:16:30,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:33,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:35,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:38,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:39,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:16:40,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:43,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:45,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:49,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:49,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:16:51,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:52,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:16:54,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:16:56,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:16:59,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:17:00,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:02,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:05,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:07,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:10,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:17:10,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:13,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:15,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:19,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:20,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:17:20,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:22,954 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:17:24,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:26,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:26,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:30,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:17:30,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:31,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:35,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:37,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:40,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:17:41,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:42,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:46,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:48,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:50,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:17:51,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:52,954 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:17:53,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:17:56,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:17:58,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:00,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:18:02,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:03,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:07,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:09,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:10,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:18:13,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:14,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:18,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:19,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:18:19,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:22,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:18:23,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:24,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:29,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:18:29,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:30,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:30,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:34,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:35,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:39,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:18:39,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:40,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:44,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:45,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:50,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:51,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:51,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:18:52,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:18:55,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:18:56,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:18:59,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:18:59,321 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:18:59,321 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:18:59,323 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:18:59,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:19:01,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:01,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:01,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:01,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:19:06,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:06,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:09,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:19:11,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:12,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:17,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:17,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:20,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:19:22,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:22,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:19:22,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:27,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:28,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:28,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:32,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:19:33,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:33,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:38,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:38,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:42,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:19:43,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:44,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:49,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:50,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:52,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:19:52,956 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:19:54,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:19:55,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:19:59,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:00,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:00,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:02,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:20:05,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:05,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:11,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:11,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:12,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:20:16,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:16,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:22,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:22,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:22,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:20:22,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:20:27,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:27,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:32,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:20:32,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:33,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:38,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:39,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:42,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:20:43,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:44,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:48,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:50,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:51,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:20:52,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:20:53,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:20:55,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:20:58,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:00,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:01,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:21:04,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:06,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:09,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:11,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:11,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:21:15,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:16,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:20,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:22,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:21:22,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:22,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:21:25,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:28,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:31,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:32,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:21:33,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:36,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:39,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:41,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:42,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:21:44,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:46,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:50,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:52,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:21:52,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:21:52,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:21:55,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:21:57,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:00,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:02,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:22:03,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:05,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:08,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:11,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:12,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:22:13,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:16,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:18,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:19,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:22:19,572 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:22:19,573 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:22:19,575 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:22:20,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:22:22,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:22,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:22:22,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:22:23,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:28,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:28,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:30,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:22:33,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:38,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:39,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:40,617 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:22:43,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:44,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:49,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:49,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:50,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:52,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:22:52,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:22:54,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:22:55,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:22:59,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:00,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:04,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:05,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:09,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:10,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:14,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:15,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:19,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:19,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:21,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:22,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:23:22,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:23:24,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:26,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:29,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:29,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:32,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:32,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:23:34,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:37,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:39,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:42,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:23:43,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:44,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:48,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:50,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:52,105 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:23:52,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:23:53,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:23:55,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:23:58,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:01,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:01,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:04,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:24:04,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:06,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:09,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:12,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:12,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:24:15,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:17,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:20,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:22,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:24:22,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:24:23,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:25,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:28,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:30,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:32,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:24:33,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:36,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:39,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:42,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:42,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:24:44,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:47,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:50,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:52,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:24:52,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:24:54,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:24:55,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:24:57,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:01,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:02,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:25:03,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:06,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:08,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:11,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:12,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:25:14,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:16,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:19,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:22,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:22,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:25:22,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:25:24,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:27,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:29,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:32,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:32,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:25:35,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:37,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:40,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:42,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:44,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:25:46,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:47,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:51,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:52,963 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:25:53,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:25:55,040 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:25:56,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:25:58,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:01,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:26:01,539 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:26:01,540 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:26:01,543 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:26:02,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:26:02,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:02,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:03,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:04,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:07,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:10,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:13,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:13,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:15,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:18,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:21,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:22,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:22,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:26:23,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:26,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:28,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:31,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:32,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:34,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:37,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:39,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:42,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:44,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:44,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:47,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:49,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:52,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:26:52,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:26:53,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:54,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:26:58,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:26:59,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:03,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:04,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:27:05,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:08,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:10,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:12,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:27:14,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:16,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:20,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:21,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:22,964 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:27:24,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:27:25,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:26,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:31,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:31,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:31,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:31,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:34,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:27:36,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:37,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:42,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:42,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:44,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:27:47,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:47,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:52,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:27:52,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:52,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:27:54,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:27:57,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:27:57,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:02,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:03,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:05,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:28:08,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:08,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:14,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:14,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:15,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:28:19,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:19,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:22,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:28:25,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:28:25,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:25,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:30,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:30,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:35,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:35,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:35,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:28:35,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:40,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:40,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:45,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:28:46,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:46,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:51,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:28:51,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:52,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:28:54,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:28:56,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:28:56,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:01,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:02,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:02,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:04,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:29:07,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:07,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:13,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:13,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:14,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:29:18,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:18,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:22,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:29:22,925 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:29:22,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:29:22,929 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:29:22,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:29:23,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:29:23,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:23,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:24,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:29:28,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:28,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:34,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:34,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:29:34,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:39,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:39,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:44,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:29:45,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:45,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:50,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:51,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:29:52,967 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:29:54,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:29:55,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:29:56,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:00,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:02,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:05,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:30:06,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:07,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:11,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:12,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:15,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:30:16,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:17,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:21,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:22,968 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:30:23,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:25,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:30:26,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:28,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:31,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:34,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:35,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:30:37,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:39,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:42,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:44,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:45,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:30:47,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:49,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:52,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:30:52,969 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:30:55,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:30:55,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:30:58,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:00,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:04,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:05,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:31:05,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:09,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:12,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:14,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:14,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:31:17,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:19,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:22,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:22,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:31:24,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:31:24,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:27,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:29,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:32,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:34,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:38,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:40,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:40,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:31:43,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:45,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:48,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:48,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:50,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:31:51,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:52,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:31:53,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:31:56,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:31:58,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:31:59,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:02,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:04,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:07,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:09,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:11,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:32:12,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:14,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:17,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:20,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:21,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:32:22,971 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:32:22,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:25,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:27,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:31,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:32:31,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:33,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:37,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:38,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:41,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:32:42,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:43,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:48,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:48,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:48,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:32:48,820 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:32:48,821 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:32:48,824 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:32:49,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:32:49,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:50,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:32:51,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:32:52,971 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:32:53,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:32:54,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:32:59,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:00,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:01,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:33:04,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:05,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:09,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:10,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:11,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:33:14,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:15,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:20,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:21,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:33:21,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:22,972 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:33:25,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:26,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:31,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:31,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:33:32,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:37,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:38,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:40,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:33:43,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:43,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:48,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:48,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:48,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:50,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:33:52,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:33:53,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:33:53,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:58,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:33:59,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:01,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:34:04,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:04,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:09,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:10,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:11,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:34:14,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:15,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:19,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:20,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:20,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:21,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:34:22,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:34:24,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:25,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:30,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:31,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:34:31,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:36,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:41,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:41,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:34:42,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:46,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:47,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:51,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:34:51,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:52,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:34:52,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:34:56,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:34:58,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:01,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:35:02,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:03,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:07,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:09,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:11,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:35:13,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:14,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:18,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:19,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:21,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:35:22,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:35:23,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:24,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:28,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:30,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:31,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:35:34,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:35,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:40,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:41,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:41,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:35:45,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:46,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:51,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:51,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:35:51,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:35:52,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:35:56,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:35:58,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:00,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:36:02,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:03,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:07,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:08,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:08,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:09,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:36:09,814 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:36:09,815 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:36:09,818 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:36:09,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:36:10,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:36:13,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:13,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:18,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:18,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:18,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:21,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:36:22,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:36:23,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:24,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:28,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:30,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:31,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:36:34,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:35,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:40,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:40,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:41,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:36:45,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:45,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:49,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:36:50,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:51,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:36:52,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:36:55,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:36:58,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:01,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:01,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:37:03,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:06,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:08,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:08,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:11,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:37:11,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:13,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:16,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:18,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:21,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:37:22,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:22,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:37:24,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:27,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:29,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:29,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:37:33,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:34,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:38,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:39,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:41,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:37:43,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:44,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:48,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:49,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:51,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:37:52,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:37:53,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:37:55,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:37:58,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:01,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:01,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:38:04,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:06,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:09,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:11,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:11,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:38:14,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:18,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:20,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:22,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:38:22,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:38:23,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:25,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:28,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:31,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:32,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:38:33,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:36,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:39,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:41,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:38:42,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:44,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:47,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:49,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:51,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:38:52,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:52,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:38:54,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:38:57,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:38:59,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:01,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:39:03,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:04,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:08,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:10,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:11,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:39:14,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:15,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:19,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:21,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:21,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:39:22,984 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:39:24,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:26,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:29,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:30,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:39:30,240 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:39:30,241 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:39:30,243 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:39:30,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:39:31,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:39:31,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:35,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:36,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:41,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:41,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:39:42,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:46,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:48,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:51,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:39:52,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:52,985 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:39:53,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:39:57,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:39:59,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:01,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:40:03,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:04,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:08,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:10,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:11,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:40:13,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:15,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:18,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:20,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:21,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:40:22,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:40:23,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:26,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:29,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:31,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:31,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:40:34,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:36,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:39,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:42,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:42,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:40:44,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:47,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:49,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:52,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:52,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:52,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:40:52,986 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:40:54,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:40:57,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:40:59,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:02,258 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:41:02,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:05,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:08,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:10,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:12,325 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:41:13,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:16,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:19,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:19,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:21,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:41:21,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:22,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:41:24,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:26,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:30,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:31,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:41:32,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:35,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:37,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:41,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:41,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:41:43,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:46,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:48,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:51,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:51,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:41:52,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:41:53,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:41:56,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:41:58,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:01,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:02,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:04,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:07,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:09,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:11,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:13,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:15,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:19,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:20,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:21,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:22,989 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:42:24,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:25,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:30,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:31,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:31,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:35,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:36,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:41,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:41,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:42,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:46,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:47,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:49,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:42:49,898 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:42:49,899 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:42:49,902 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:42:50,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:42:51,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:51,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:52,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:42:52,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:42:52,989 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:42:56,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:42:57,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:02,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:43:02,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:03,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:07,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:08,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:12,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:13,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:17,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:19,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:20,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:43:22,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:22,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:43:24,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:28,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:30,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:43:30,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:33,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:35,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:39,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:40,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:43:41,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:44,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:46,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:49,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:50,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:43:51,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:52,991 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:43:56,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:43:56,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:43:57,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:01,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:02,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:06,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:06,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:07,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:09,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:11,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:12,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:16,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:18,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:19,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:21,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:22,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:44:23,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:26,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:27,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:29,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:31,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:34,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:37,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:37,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:39,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:42,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:44,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:47,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:48,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:50,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:52,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:44:53,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:44:55,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:44:58,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:44:59,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:01,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:05,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:07,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:08,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:45:10,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:12,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:16,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:17,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:20,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:45:21,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:22,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:22,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:45:26,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:27,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:30,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:45:31,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:32,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:37,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:37,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:38,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:40,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:45:42,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:43,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:48,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:48,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:45:49,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:52,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:45:53,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:45:54,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:45:58,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:00,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:00,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:03,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:05,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:08,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:08,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:10,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:11,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:13,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:16,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:18,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:46:18,836 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:46:18,836 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:46:18,839 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:46:19,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:19,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:46:20,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:21,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:22,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:22,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:46:24,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:27,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:29,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:32,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:35,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:36,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:38,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:40,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:43,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:44,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:45,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:49,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:50,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:52,996 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:46:55,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:46:55,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:46:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:46:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:00,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:00,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:05,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:05,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:47:05,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:10,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:10,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:16,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:47:16,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:16,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:21,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:22,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:23,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:23,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:23,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:47:26,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:47:26,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:27,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:31,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:33,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:36,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:47:36,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:38,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:41,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:43,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:46,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:47:47,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:48,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:52,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:52,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:47:54,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:54,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:54,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:47:56,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:47:58,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:47:59,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:03,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:05,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:06,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:48:09,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:10,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:14,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:15,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:16,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:48:20,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:21,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:23,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:48:25,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:26,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:26,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:48:30,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:31,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:36,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:36,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:48:36,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:41,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:41,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:46,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:46,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:48:47,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:51,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:53,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:48:53,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:48:56,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:48:56,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:48:58,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:01,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:04,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:06,797 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:49:07,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:09,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:12,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:15,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:16,865 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:49:18,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:20,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:23,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:49:24,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:25,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:49:26,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:29,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:31,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:34,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:36,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:49:37,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:40,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:42,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:45,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:46,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:49:46,943 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:49:46,944 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:49:46,947 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:49:47,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:49:47,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:48,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:49:50,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:52,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:53,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:49:56,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:49:57,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:49:58,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:50:01,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:02,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:06,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:08,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:50:08,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:12,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:13,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:17,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:18,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:50:18,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:22,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:23,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:50:23,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:28,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:50:28,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:28,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:33,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:34,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:38,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:50:38,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:40,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:43,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:46,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:48,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:50:49,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:51,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:53,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:50:55,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:50:56,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:56,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:50:58,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:51:00,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:01,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:05,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:07,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:08,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:51:11,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:12,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:16,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:18,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:18,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:51:22,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:23,004 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:51:23,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:28,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:28,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:28,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:51:33,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:33,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:38,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:38,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:51:39,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:43,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:44,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:48,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:51:49,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:50,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:53,004 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:51:55,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:51:55,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:51:58,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:00,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:00,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:05,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:06,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:09,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:11,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:11,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:17,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:17,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:19,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:22,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:22,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:23,005 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:52:27,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:28,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:28,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:32,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:33,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:38,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:38,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:38,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:43,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:43,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:48,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:48,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:49,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:53,005 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:52:54,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:52:55,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:52:58,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:52:59,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:00,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:04,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:06,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:06,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:53:06,227 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:53:06,228 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:53:06,230 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:53:06,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:53:08,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:53:10,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:11,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:15,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:16,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:16,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:18,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:53:20,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:21,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:23,006 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:53:26,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:26,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:26,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:28,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:53:31,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:32,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:37,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:37,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:38,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:53:42,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:42,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:46,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:53:47,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:47,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:52,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:53,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:53:54,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:53:56,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:53:58,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:53:59,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:03,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:05,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:08,879 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:54:09,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:10,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:14,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:15,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:18,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:54:20,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:20,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:23,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:54:25,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:26,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:29,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:54:30,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:31,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:36,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:36,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:39,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:54:41,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:41,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:47,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:54:47,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:47,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:52,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:52,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:53,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:54:57,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:54:58,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:54:59,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:55:03,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:03,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:07,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:55:08,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:08,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:13,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:14,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:14,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:16,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:55:19,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:19,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:23,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:55:25,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:25,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:26,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:55:30,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:30,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:35,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:36,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:36,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:55:40,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:41,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:45,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:46,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:46,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:55:50,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:51,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:53,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:55:56,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:55:56,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:56,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:55:58,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:01,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:02,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:07,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:08,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:08,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:12,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:13,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:16,867 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:17,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:18,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:22,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:23,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:56:24,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:25,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:56:25,788 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:56:25,789 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:56:25,792 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:56:25,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:56:26,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:27,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:28,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:29,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:34,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:37,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:38,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:39,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:43,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:44,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:47,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:48,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:49,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:53,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:56:54,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:56:55,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:56:59,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:56:59,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:00,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:04,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:05,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:09,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:57:10,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:10,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:15,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:16,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:16,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:19,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:57:21,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:21,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:23,015 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:57:26,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:26,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:27,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:29,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:57:32,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:32,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:38,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:38,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:39,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:57:43,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:43,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:48,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:57:49,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:49,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:53,015 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:57:55,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:57:55,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:57:58,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:00,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:00,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:05,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:05,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:08,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:10,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:10,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:15,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:15,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:15,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:18,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:20,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:21,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:23,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:58:26,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:26,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:27,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:28,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:31,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:32,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:37,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:37,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:39,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:42,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:42,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:48,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:48,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:49,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:53,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:58:53,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:54,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:58:58,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:58:59,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:58:59,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:04,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:04,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:09,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:09,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:10,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:15,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:15,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:15,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:19,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:20,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:20,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:23,019 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:59:26,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:26,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:27,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:31,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:31,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:36,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:36,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:37,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:41,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:41,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:45,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 17:59:45,517 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 17:59:45,517 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 17:59:45,521 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 17:59:46,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 17:59:46,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:47,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:47,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:47,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:52,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:53,019 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 17:59:54,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 17:59:57,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 17:59:58,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 17:59:59,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:03,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:04,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:07,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:00:08,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:09,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:13,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:15,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:15,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:17,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:00:19,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:20,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:23,020 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:00:25,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:26,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:26,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:00:30,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:31,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:36,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:36,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:00:37,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:41,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:42,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:46,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:00:47,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:47,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:52,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:52,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:00:53,021 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:00:56,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:00:57,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:00:58,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:02,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:03,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:07,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:01:08,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:08,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:13,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:13,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:18,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:19,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:23,021 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:01:23,181 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:01:24,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:24,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:29,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:30,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:30,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:31,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:01:34,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:35,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:39,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:40,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:43,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:01:45,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:45,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:50,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:51,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:01:53,022 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:01:53,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:01:56,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:01:56,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:01,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:02,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:03,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:02:06,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:07,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:12,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:12,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:13,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:02:17,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:17,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:23,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:02:23,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:23,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:23,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:02:28,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:28,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:33,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:33,676 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:02:34,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:38,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:39,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:43,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:02:44,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:44,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:49,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:50,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:02:53,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:02:53,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:02:55,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:02:55,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:00,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:01,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:01,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:02,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:03:05,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:06,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:10,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:03:10,659 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:03:10,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:03:10,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:10,663 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:03:11,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:03:11,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:13,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:03:15,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:17,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:20,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:22,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:23,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:03:23,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:03:26,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:27,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:31,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:32,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:33,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:03:36,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:37,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:42,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:43,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:03:43,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:47,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:48,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:53,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:03:53,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:03:53,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:03:54,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:03:58,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:00,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:00,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:03,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:03,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:04:05,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:08,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:10,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:13,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:04:14,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:15,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:19,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:21,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:23,024 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:04:23,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:04:25,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:26,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:30,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:31,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:33,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:04:35,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:36,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:40,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:42,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:43,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:04:46,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:47,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:51,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:53,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:04:53,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:04:53,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:04:57,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:04:58,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:02,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:03,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:05:04,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:07,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:09,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:13,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:13,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:05:14,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:18,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:21,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:23,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:05:23,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:05:24,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:26,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:29,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:31,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:34,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:05:34,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:36,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:39,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:42,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:44,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:05:45,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:47,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:50,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:52,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:05:53,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:05:53,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:05:56,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:05:57,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:01,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:03,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:06:03,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:06,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:08,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:12,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:13,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:06:13,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:17,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:18,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:23,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:06:23,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:23,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:06:23,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:28,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:28,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:32,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:06:32,114 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:06:32,115 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:06:32,117 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:06:32,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:06:32,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:06:33,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:33,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:06:34,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:38,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:39,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:43,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:44,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:44,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:45,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:06:48,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:49,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:53,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:06:54,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:54,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:06:59,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:06:59,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:04,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:05,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:07,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:07:09,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:10,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:14,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:15,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:19,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:20,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:23,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:07:25,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:25,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:30,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:30,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:35,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:36,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:37,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:07:40,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:41,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:45,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:47,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:47,658 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:07:51,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:53,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:07:57,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:07:57,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:07:57,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:08:02,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:02,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:07,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:07,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:08:07,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:12,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:12,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:18,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:18,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:19,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:08:23,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:08:23,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:24,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:28,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:29,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:30,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:08:33,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:34,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:38,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:39,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:40,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:08:43,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:44,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:48,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:50,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:08:50,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:53,029 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:08:53,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:56,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:08:58,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:08:58,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:00,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:09:01,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:03,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:06,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:09,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:10,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:09:11,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:14,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:16,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:20,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:21,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:09:21,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:23,030 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:09:25,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:27,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:30,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:31,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:09:32,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:35,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:37,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:41,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:41,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:09:42,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:46,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:47,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:51,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:51,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:09:52,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:09:53,032 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:09:56,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:09:58,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:02,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:03,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:03,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:10:07,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:08,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:13,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:13,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:13,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:15,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:10:18,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:18,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:23,032 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:10:24,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:24,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:25,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:10:29,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:29,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:34,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:34,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:10:35,208 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:10:35,209 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:10:35,212 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:10:36,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:10:36,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:10:37,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:10:39,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:40,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:45,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:45,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:48,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:10:50,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:10:50,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:53,032 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:10:56,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:10:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:00,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:11:01,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:01,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:06,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:10,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:11:11,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:11,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:16,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:17,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:21,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:22,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:11:22,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:23,033 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:11:27,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:27,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:32,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:33,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:33,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:11:37,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:38,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:41,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:11:43,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:44,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:48,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:49,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:53,033 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:11:53,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:11:54,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:11:54,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:11:59,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:01,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:03,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:12:04,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:06,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:09,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:11,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:14,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:12:14,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:16,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:19,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:22,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:23,034 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:12:24,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:12:25,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:27,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:30,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:32,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:34,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:12:35,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:37,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:40,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:43,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:44,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:12:45,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:48,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:50,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:53,035 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:12:53,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:12:54,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:12:56,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:12:58,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:01,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:03,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:04,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:06,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:08,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:11,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:13,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:14,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:17,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:18,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:22,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:23,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:13:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:26,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:27,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:29,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:32,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:34,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:36,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:38,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:39,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:43,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:44,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:44,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:46,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:48,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:49,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:53,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:13:54,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:13:55,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:13:56,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:58,844 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:13:59,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:00,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:04,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:05,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:07,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:14:07,029 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:14:07,029 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:14:07,032 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:14:07,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:14:07,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:14:09,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:14:10,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:10,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:15,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:16,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:16,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:19,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:14:20,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:21,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:23,037 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:14:26,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:27,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:30,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:14:31,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:32,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:36,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:37,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:40,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:14:41,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:42,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:46,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:48,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:51,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:52,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:14:53,038 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:14:53,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:57,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:14:58,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:58,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:14:58,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:02,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:02,293 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:15:03,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:07,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:09,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:12,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:15:13,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:14,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:18,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:19,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:22,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:15:23,038 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:15:24,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:24,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:29,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:30,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:30,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:32,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:15:34,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:35,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:39,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:40,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:42,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:15:45,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:45,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:50,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:51,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:15:52,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:15:53,039 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:15:55,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:15:56,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:00,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:02,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:02,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:02,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:16:05,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:07,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:10,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:12,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:12,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:16:15,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:17,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:20,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:22,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:23,040 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:16:24,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:16:26,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:27,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:31,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:32,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:32,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:34,065 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:16:36,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:37,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:41,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:43,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:44,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:16:47,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:48,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:52,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:53,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:16:53,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:16:56,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:16:57,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:16:58,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:02,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:03,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:07,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:08,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:10,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:17:12,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:13,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:17,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:18,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:18,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:22,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:17:23,041 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:17:23,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:23,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:28,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:29,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:32,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:17:33,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:34,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:38,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:39,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:41,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:17:41,011 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:17:41,012 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:17:41,017 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:17:41,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:17:42,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:17:43,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:17:44,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:44,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:49,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:51,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:53,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:17:54,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:17:54,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:17:56,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:17:59,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:01,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:01,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:04,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:04,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:18:06,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:09,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:11,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:14,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:14,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:18:16,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:19,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:21,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:23,042 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:18:25,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:18:25,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:26,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:30,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:31,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:31,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:31,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:35,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:18:35,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:36,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:40,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:42,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:46,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:18:46,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:47,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:51,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:52,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:18:53,044 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:18:56,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:18:57,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:18:57,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:02,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:03,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:03,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:06,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:19:07,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:08,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:12,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:14,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:16,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:19:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:19,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:23,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:19:24,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:24,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:28,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:19:29,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:29,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:34,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:35,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:35,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:38,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:19:39,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:40,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:44,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:45,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:48,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:19:50,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:50,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:53,045 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:19:56,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:19:56,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:19:58,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:20:01,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:01,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:06,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:07,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:07,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:10,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:20:12,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:12,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:17,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:17,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:21,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:20:22,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:22,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:23,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:20:28,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:28,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:31,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:20:33,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:33,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:38,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:38,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:41,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:20:43,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:44,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:49,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:49,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:51,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:20:53,046 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:20:54,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:20:55,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:20:59,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:00,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:01,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:04,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:05,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:08,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:21:08,999 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:21:09,000 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:21:09,002 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:21:09,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:21:10,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:11,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:11,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:11,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:16,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:16,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:20,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:21,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:21,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:21,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:23,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:21:26,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:27,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:32,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:32,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:32,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:37,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:37,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:42,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:43,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:43,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:48,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:48,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:50,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:21:53,047 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:21:53,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:54,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:21:59,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:21:59,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:00,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:22:04,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:04,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:09,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:09,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:10,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:22:14,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:14,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:19,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:20,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:20,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:22:23,048 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:22:25,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:26,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:30,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:31,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:22:31,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:35,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:36,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:41,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:22:41,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:42,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:47,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:47,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:51,137 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:22:52,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:52,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:22:53,048 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:22:58,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:22:58,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:01,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:23:03,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:03,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:08,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:08,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:09,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:09,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:23:13,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:14,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:18,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:18,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:19,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:23:20,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:23,049 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:23:23,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:26,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:29,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:31,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:23:31,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:34,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:36,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:40,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:41,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:23:42,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:45,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:47,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:50,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:23:50,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:52,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:23:53,049 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:23:57,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:23:58,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:00,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:02,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:02,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:04,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:07,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:09,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:12,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:14,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:14,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:17,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:19,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:23,050 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:24:23,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:23,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:24,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:25,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:28,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:30,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:32,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:24:32,558 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:24:32,559 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:24:32,561 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:24:32,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:24:33,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:33,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:34,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:35,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:39,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:40,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:45,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:45,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:45,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:50,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:50,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:24:53,050 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:24:55,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:24:55,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:24:56,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:00,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:01,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:05,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:25:05,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:07,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:11,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:12,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:15,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:25:16,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:18,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:22,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:22,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:23,051 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:25:24,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:25,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:25:27,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:29,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:32,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:34,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:35,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:25:37,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:39,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:43,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:45,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:45,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:25:48,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:50,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:53,051 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:25:53,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:25:55,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:25:56,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:25:58,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:01,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:04,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:05,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:26:06,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:09,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:11,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:14,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:17,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:26:17,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:19,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:22,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:23,053 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:26:25,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:26,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:26:28,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:30,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:33,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:35,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:35,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:36,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:26:39,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:40,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:44,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:46,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:46,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:26:49,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:51,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:26:53,055 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:26:55,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:26:57,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:26:57,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:00,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:03,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:06,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:07,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:27:08,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:11,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:13,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:17,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:17,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:27:18,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:22,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:23,054 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:27:23,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:28,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:28,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:33,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:33,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:38,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:39,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:43,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:44,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:48,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:49,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:27:51,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:27:53,055 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:27:54,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:27:55,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:00,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:00,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:01,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:05,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:05,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:09,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:10,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:11,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:15,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:17,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:18,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:28:18,574 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:28:18,575 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:28:18,577 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:28:18,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:28:19,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:20,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:20,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:22,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:23,055 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:28:26,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:28,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:29,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:31,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:33,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:36,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:38,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:38,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:41,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:42,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:43,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:47,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:48,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:48,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:51,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:28:53,056 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:28:53,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:28:54,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:28:58,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:00,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:00,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:29:03,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:05,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:08,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:11,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:29:11,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:13,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:16,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:18,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:21,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:29:21,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:23,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:29:24,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:28,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:29,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:31,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:29:33,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:34,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:39,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:39,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:43,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:29:44,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:44,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:49,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:50,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:50,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:29:51,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:29:53,057 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:29:55,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:29:55,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:00,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:00,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:03,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:30:05,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:05,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:11,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:30:11,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:11,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:16,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:16,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:21,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:22,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:22,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:23,057 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:30:23,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:30:27,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:27,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:32,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:32,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:33,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:30:37,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:37,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:43,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:43,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:43,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:30:48,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:48,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:53,058 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:30:53,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:53,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:30:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:54,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:30:58,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:30:59,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:03,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:03,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:04,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:08,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:09,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:13,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:14,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:14,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:19,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:19,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:23,058 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:31:24,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:25,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:25,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:30,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:31,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:33,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:35,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:36,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:40,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:42,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:43,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:31:43,602 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:31:43,603 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:31:43,606 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:31:44,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:31:44,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:45,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:46,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:47,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:51,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:53,059 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:31:53,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:53,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:31:55,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:31:56,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:31:58,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:01,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:04,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:05,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:32:07,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:09,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:12,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:14,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:15,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:32:18,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:21,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:23,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:32:23,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:23,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:32:26,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:28,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:32,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:33,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:32:33,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:37,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:38,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:42,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:43,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:47,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:48,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:49,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:32:52,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:53,060 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:32:54,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:58,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:32:59,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:32:59,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:33:03,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:04,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:09,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:09,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:09,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:33:14,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:14,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:19,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:19,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:19,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:33:23,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:33:24,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:25,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:30,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:30,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:33:30,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:35,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:35,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:40,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:33:40,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:41,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:45,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:46,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:51,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:51,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:33:52,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:33:53,062 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:33:56,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:33:57,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:01,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:02,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:03,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:34:06,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:07,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:12,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:12,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:12,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:13,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:34:17,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:17,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:22,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:22,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:23,063 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:34:23,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:23,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:34:28,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:28,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:33,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:33,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:34:33,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:38,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:38,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:43,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:44,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:45,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:34:48,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:49,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:53,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:34:54,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:54,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:54,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:34:57,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:34:59,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:34:59,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:04,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:05,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:07,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:35:09,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:10,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:14,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:16,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:17,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:35:17,606 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:35:17,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:35:17,610 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:35:17,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:35:18,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:35:19,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:35:20,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:21,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:23,064 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:35:26,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:27,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:27,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:29,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:35:31,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:32,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:36,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:38,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:42,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:35:42,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:43,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:47,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:49,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:52,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:35:53,065 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:35:53,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:35:54,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:35:58,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:00,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:03,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:04,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:36:05,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:08,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:11,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:11,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:13,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:14,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:36:16,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:18,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:22,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:23,066 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:36:24,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:25,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:36:27,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:29,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:32,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:35,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:35,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:36:37,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:40,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:44,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:45,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:47,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:36:49,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:50,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:53,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:36:54,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:54,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:36:56,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:36:57,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:36:59,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:01,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:05,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:06,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:07,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:37:10,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:11,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:15,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:16,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:17,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:37:20,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:22,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:23,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:37:26,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:26,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:27,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:30,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:37:31,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:32,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:37,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:37,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:40,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:37:42,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:43,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:48,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:48,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:50,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:37:53,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:53,068 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:37:54,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:37:58,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:58,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:37:59,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:00,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:03,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:04,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:08,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:09,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:12,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:13,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:14,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:19,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:19,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:22,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:23,069 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:38:24,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:25,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:30,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:30,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:34,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:35,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:35,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:40,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:41,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:44,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:46,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:46,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:51,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:52,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:38:53,070 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:38:53,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:38:53,762 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:38:53,763 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:38:53,767 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:38:54,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:38:54,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:55,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:38:56,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:38:57,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:01,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:02,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:05,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:39:07,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:07,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:12,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:13,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:13,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:15,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:39:17,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:18,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:22,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:23,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:39:24,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:28,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:39:28,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:29,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:33,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:35,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:38,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:39:38,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:40,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:43,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:45,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:45,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:48,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:39:48,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:50,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:53,072 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:39:54,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:39:55,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:39:58,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:40:00,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:00,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:05,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:06,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:08,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:40:10,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:11,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:15,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:17,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:18,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:40:20,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:22,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:23,073 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:40:26,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:28,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:28,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:30,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:40:31,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:33,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:36,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:38,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:40,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:40:41,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:43,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:46,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:48,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:52,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:52,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:40:53,074 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:40:53,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:58,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:40:59,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:40:59,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:02,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:41:03,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:04,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:08,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:10,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:11,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:41:13,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:15,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:18,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:21,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:23,075 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:41:23,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:41:24,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:26,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:29,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:31,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:34,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:36,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:39,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:42,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:42,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:44,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:46,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:41:47,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:49,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:53,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:41:53,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:54,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:41:56,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:41:58,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:41:59,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:03,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:04,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:06,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:42:08,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:10,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:14,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:14,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:14,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:15,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:16,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:42:19,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:20,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:23,077 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:42:25,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:26,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:28,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:42:30,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:31,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:36,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:36,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:37,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:42:37,968 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:42:37,968 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:42:37,972 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:42:38,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:42:39,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:42:40,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:42:41,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:41,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:46,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:47,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:50,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:42:52,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:52,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:42:53,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:42:57,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:42:58,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:00,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:43:02,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:03,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:08,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:08,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:10,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:43:13,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:13,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:18,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:18,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:18,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:20,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:43:23,079 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:43:23,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:24,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:29,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:29,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:32,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:43:34,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:34,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:39,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:40,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:42,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:43:44,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:45,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:49,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:50,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:43:53,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:43:53,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:43:55,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:43:55,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:00,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:01,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:01,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:01,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:04,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:44:05,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:06,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:10,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:11,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:14,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:44:16,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:16,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:21,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:21,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:23,080 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:44:24,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:44:26,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:27,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:32,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:32,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:36,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:44:37,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:38,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:43,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:43,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:46,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:44:48,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:48,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:53,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:44:53,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:54,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:44:56,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:44:58,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:44:59,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:04,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:04,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:04,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:06,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:45:09,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:09,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:14,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:14,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:18,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:45:19,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:20,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:23,082 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:45:25,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:26,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:28,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:45:30,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:31,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:36,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:37,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:40,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:45:41,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:42,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:46,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:47,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:47,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:51,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:45:52,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:52,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:45:53,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:45:58,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:45:58,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:01,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:46:03,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:03,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:08,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:08,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:10,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:46:10,203 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:46:10,222 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:46:10,226 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:46:10,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:46:11,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:46:13,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:14,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:19,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:19,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:19,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:22,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:46:23,083 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:46:24,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:25,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:30,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:30,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:32,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:46:35,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:35,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:40,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:41,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:44,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:46:45,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:46,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:50,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:52,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:46:53,084 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:46:54,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:46:56,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:46:57,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:01,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:02,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:02,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:02,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:04,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:47:07,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:07,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:12,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:13,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:16,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:47:17,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:18,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:22,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:23,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:47:25,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:27,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:28,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:47:30,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:32,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:35,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:38,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:38,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:47:40,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:43,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:46,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:48,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:49,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:47:51,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:53,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:47:54,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:47:57,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:47:59,140 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:47:59,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:02,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:04,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:08,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:09,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:11,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:48:13,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:14,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:18,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:18,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:18,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:19,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:21,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:48:23,086 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:48:23,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:25,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:29,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:30,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:33,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:48:34,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:35,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:40,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:40,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:42,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:48:45,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:46,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:50,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:51,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:53,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:48:56,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:48:57,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:48:58,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:01,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:02,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:06,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:06,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:07,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:08,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:11,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:13,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:17,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:18,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:20,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:22,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:23,087 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:49:24,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:29,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:29,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:32,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:34,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:34,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:39,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:39,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:42,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:44,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:45,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:50,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:51,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:51,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:51,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:52,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 18:49:52,331 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 18:49:52,331 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 18:49:52,334 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 18:49:53,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 18:49:53,088 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:49:54,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:55,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:49:56,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:49:56,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:49:57,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 18:50:01,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:01,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:06,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:06,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:11,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:11,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:16,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:16,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:21,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:21,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:23,089 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:50:26,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:27,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:31,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:32,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:36,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:37,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:41,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:42,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:46,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:47,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:51,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:52,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:50:53,090 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:50:56,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:50:58,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:01,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:03,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:06,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:08,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:11,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:13,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:16,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:18,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:21,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:23,090 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:51:24,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:26,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:29,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:31,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:34,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:36,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:39,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:41,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:44,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:46,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:49,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:51,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:51:53,091 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:51:55,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:51:56,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:00,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:01,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:05,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:06,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:10,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:11,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:15,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:16,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:20,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:21,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:23,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:52:26,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:26,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:31,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:31,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:36,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:36,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:41,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:41,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:46,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:46,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:51,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:52:51,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:53,092 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:52:56,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:52:57,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:01,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:02,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:06,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:07,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:11,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:12,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:16,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:17,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:21,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:22,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:23,093 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:53:26,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:28,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:31,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:33,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:36,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:38,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:41,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:43,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:46,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:48,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:51,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:53,093 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:53:54,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:53:56,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:53:59,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:01,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:04,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:06,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:09,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:11,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:14,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:16,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:19,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:21,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:23,094 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:54:25,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:26,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:30,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:31,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:35,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:36,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:40,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:41,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:45,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:46,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:50,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:51,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:54:53,094 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:54:56,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:54:56,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:01,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:01,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:06,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:06,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:11,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:16,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:16,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:21,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:21,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:23,094 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:55:26,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:27,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:31,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:32,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:36,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:37,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:41,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:42,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:46,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:47,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:51,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:52,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:55:53,095 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:55:56,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:55:58,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:01,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:03,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:06,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:08,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:11,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:13,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:16,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:18,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:21,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:23,096 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:56:24,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:26,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:29,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:31,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:34,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:36,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:39,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:41,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:44,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:46,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:49,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:51,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:56:53,096 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:56:55,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:56:56,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:00,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:01,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:05,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:06,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:10,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:11,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:15,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:16,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:20,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:21,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:23,107 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:57:25,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:26,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:30,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:31,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:35,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:36,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:40,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:41,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:45,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:46,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:50,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:51,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:57:53,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:57:56,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:57:56,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:01,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:01,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:06,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:06,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:11,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:11,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:16,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:16,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:21,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:21,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:23,097 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:58:26,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:27,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:31,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:32,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:36,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:37,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:41,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:42,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:46,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:47,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:51,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:52,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:58:53,098 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:58:56,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:58:58,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:01,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:03,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:06,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:08,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:11,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:13,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:16,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:18,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:21,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:23,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:59:24,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:26,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:29,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:31,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:34,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:36,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:39,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:41,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:44,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:46,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:49,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:51,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 18:59:53,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 18:59:55,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 18:59:56,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:00,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:01,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:05,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:06,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:10,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:11,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:15,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:16,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:20,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:21,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:23,099 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:00:26,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:26,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:31,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:31,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:36,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:36,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:41,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:41,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:46,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:46,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:51,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:00:51,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:53,100 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:00:56,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:00:57,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:01,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:02,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:06,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:07,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:11,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:12,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:16,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:17,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:21,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:22,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:23,100 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:01:26,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:28,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:31,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:33,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:33,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:01:33,368 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:01:33,368 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:01:33,371 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:01:33,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:01:34,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:01:35,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:01:36,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:38,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:41,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:43,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:46,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:48,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:51,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:53,101 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:01:54,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:01:56,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:01:59,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:01,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:03,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:02:04,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:06,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:09,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:12,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:14,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:17,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:19,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:22,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:23,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:02:25,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:28,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:30,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:34,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:35,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:40,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:42,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:44,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:02:45,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:02:45,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:48,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:51,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:02:51,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:02:53,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:02:55,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:02:57,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:02,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:02,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:07,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:09,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:09,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:11,183 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:03:12,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:14,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:17,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:19,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:22,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:23,102 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:03:24,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:28,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:29,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:33,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:34,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:38,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:39,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:43,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:44,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:48,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:49,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:53,103 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:03:54,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:54,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:03:59,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:03:59,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:04,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:09,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:09,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:14,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:14,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:18,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:04:19,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:20,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:23,104 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:04:25,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:25,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:25,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:28,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:04:30,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:31,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:36,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:36,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:39,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:04:41,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:41,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:46,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:47,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:49,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:04:51,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:52,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:53,104 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:04:57,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:04:57,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:57,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:04:59,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:05:02,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:02,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:07,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:08,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:09,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:05:12,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:13,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:17,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:18,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:22,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:05:23,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:05:23,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:23,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:28,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:29,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:29,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:32,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:05:33,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:34,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:38,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:39,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:42,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:05:43,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:44,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:48,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:49,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:52,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:05:53,105 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:05:54,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:54,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:59,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:05:59,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:05:59,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:02,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:06:04,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:05,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:10,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:10,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:12,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:06:15,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:15,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:20,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:21,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:23,106 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:06:24,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:06:26,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:26,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:31,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:32,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:34,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:06:36,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:37,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:41,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:42,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:44,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:06:47,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:48,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:52,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:53,106 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:06:53,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:06:54,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:06:58,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:06:58,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:03,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:03,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:04,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:07:08,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:08,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:13,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:14,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:14,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:15,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:07:19,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:19,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:23,110 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:07:24,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:24,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:25,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:07:29,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:29,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:34,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:34,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:36,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:07:39,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:39,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:44,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:45,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:45,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:45,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:07:45,471 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:07:45,472 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:07:45,475 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:07:46,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:07:47,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:07:48,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:07:50,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:50,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:53,109 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:07:55,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:07:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:07:58,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:08:00,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:01,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:05,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:06,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:08,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:08:10,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:11,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:15,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:15,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:16,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:18,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:08:20,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:21,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:23,109 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:08:26,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:27,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:30,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:08:31,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:32,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:37,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:37,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:40,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:08:42,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:42,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:47,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:48,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:52,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:08:53,110 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:08:53,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:53,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:08:58,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:08:59,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:03,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:04,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:09:04,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:08,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:11,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:13,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:14,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:09:16,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:18,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:21,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:23,111 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:09:24,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:09:24,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:26,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:29,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:32,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:35,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:36,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:09:37,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:40,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:42,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:45,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:47,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:48,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:09:50,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:52,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:09:53,112 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:09:56,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:09:58,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:09:58,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:01,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:03,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:06,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:08,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:10,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:10:12,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:13,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:17,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:19,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:20,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:10:22,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:23,112 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:10:24,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:28,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:30,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:30,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:10:33,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:35,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:38,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:40,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:42,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:10:43,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:45,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:48,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:51,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:52,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:10:53,113 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:10:54,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:10:56,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:10:59,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:01,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:01,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:04,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:11:04,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:06,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:09,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:12,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:14,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:11:15,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:17,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:20,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:23,115 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:11:23,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:24,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:11:24,647 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:11:24,648 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:11:24,650 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:11:25,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:11:25,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:27,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:11:28,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:30,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:34,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:34,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:35,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:39,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:11:39,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:40,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:44,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:45,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:49,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:50,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:11:50,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:11:53,115 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:11:54,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:11:56,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:00,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:00,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:12:01,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:05,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:06,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:11,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:11,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:12,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:12:16,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:16,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:22,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:22,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:22,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:12:23,116 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:12:27,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:28,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:32,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:32,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:12:34,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:37,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:39,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:42,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:44,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:12:44,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:47,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:49,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:52,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:12:53,118 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:12:55,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:12:56,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:12:57,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:00,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:03,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:03,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:05,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:06,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:13:08,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:10,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:13,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:15,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:16,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:13:18,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:20,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:23,119 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:13:23,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:26,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:27,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:13:28,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:31,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:34,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:37,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:37,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:13:39,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:42,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:45,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:47,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:13:47,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:50,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:52,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:53,120 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:13:56,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:13:58,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:13:59,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:14:01,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:03,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:07,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:07,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:09,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:11,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:14:12,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:14,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:17,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:19,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:20,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:14:22,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:23,120 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:14:25,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:27,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:30,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:14:31,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:32,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:36,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:38,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:40,640 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:14:41,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:43,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:46,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:48,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:51,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:52,728 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:14:53,121 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:14:53,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:14:57,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:14:59,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:00,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:15:00,216 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:15:00,217 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:15:00,220 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:15:00,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:15:01,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:02,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:02,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:04,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:07,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:09,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:09,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:12,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:12,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:14,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:17,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:19,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:22,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:23,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:23,122 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:15:24,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:28,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:30,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:33,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:35,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:35,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:38,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:42,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:43,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:47,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:47,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:49,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:53,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:15:54,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:54,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:15:55,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:15:57,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:15:59,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:00,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:04,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:05,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:07,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:16:09,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:10,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:14,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:15,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:19,493 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:16:19,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:21,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:23,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:16:25,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:25,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:26,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:29,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:16:30,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:31,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:36,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:36,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:36,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:38,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:16:41,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:41,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:46,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:46,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:50,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:16:51,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:52,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:53,123 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:16:57,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:16:58,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:16:58,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:17:02,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:03,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:08,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:08,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:17:13,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:13,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:18,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:19,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:17:19,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:23,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:17:24,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:26,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:29,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:17:29,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:31,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:34,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:37,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:37,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:39,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:17:40,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:42,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:45,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:47,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:50,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:51,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:17:52,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:53,124 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:17:56,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:17:58,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:17:58,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:01,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:01,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:03,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:06,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:08,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:11,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:13,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:13,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:16,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:20,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:21,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:23,125 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:18:23,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:25,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:27,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:31,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:32,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:32,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:18:32,408 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:18:32,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:18:32,411 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:18:32,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:18:33,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:35,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:36,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:37,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:41,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:42,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:45,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:46,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:48,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:52,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:52,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:53,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:18:53,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:18:55,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:18:57,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:18:58,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:02,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:03,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:07,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:08,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:12,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:19:13,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:14,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:18,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:19,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:22,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:19:23,127 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:19:23,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:24,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:29,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:29,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:34,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:35,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:19:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:39,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:45,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:47,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:19:50,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:50,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:53,129 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:19:55,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:19:56,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:19:57,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:20:00,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:01,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:06,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:06,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:07,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:20:11,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:11,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:16,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:16,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:21,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:23,130 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:20:26,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:27,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:32,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:32,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:35,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:20:37,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:37,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:42,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:42,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:45,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:20:47,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:48,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:52,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:52,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:53,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:20:53,131 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:20:57,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:20:57,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:20:58,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:03,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:03,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:07,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:21:08,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:08,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:13,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:13,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:15,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:21:19,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:19,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:23,132 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:21:25,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:25,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:25,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:21:30,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:30,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:35,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:21:35,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:36,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:40,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:41,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:46,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:47,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:47,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:21:51,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:52,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:21:53,134 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:21:57,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:21:57,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:21:57,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:02,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:03,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:07,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:08,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:13,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:14,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:17,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:19,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:19,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:23,135 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:22:24,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:22:24,284 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:22:24,285 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:22:24,288 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:22:24,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:22:25,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:25,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:25,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:25,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:26,412 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:30,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:35,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:35,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:36,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:40,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:41,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:45,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:46,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:46,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:51,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:51,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:22:53,136 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:22:55,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:22:57,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:22:57,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:02,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:02,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:05,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:23:07,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:07,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:12,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:13,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:15,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:23:18,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:18,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:23,138 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:23:24,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:24,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:24,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:25,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:23:29,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:29,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:34,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:35,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:35,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:23:39,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:40,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:44,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:45,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:48,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:23:50,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:50,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:53,139 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:23:55,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:23:56,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:23:57,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:24:00,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:01,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:05,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:06,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:06,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:09,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:24:11,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:11,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:16,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:16,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:19,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:24:21,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:21,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:23,139 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:24:27,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:27,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:29,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:24:32,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:32,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:37,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:38,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:41,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:24:43,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:43,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:48,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:48,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:51,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:24:53,140 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:24:53,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:54,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:59,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:24:59,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:24:59,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:03,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:25:04,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:04,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:09,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:11,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:13,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:25:15,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:16,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:20,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:21,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:23,140 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:25:23,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:25:26,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:26,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:31,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:31,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:36,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:25:36,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:37,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:42,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:42,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:46,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:25:47,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:47,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:52,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:25:52,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:53,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:25:54,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:25:54,266 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:25:54,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:25:54,269 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:25:55,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:25:56,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:25:57,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:25:58,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:02,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:03,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:08,316 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:26:08,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:09,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:14,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:15,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:15,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:17,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:26:19,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:20,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:23,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:26:25,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:26,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:29,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:26:30,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:31,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:35,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:37,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:39,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:26:41,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:42,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:46,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:48,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:49,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:26:51,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:53,143 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:26:53,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:57,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:26:58,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:26:59,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:27:02,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:03,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:07,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:08,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:11,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:27:13,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:13,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:18,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:19,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:22,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:27:23,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:23,144 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:27:24,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:28,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:30,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:32,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:27:33,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:35,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:38,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:40,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:42,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:27:44,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:45,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:49,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:50,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:52,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:27:53,145 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:27:54,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:27:55,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:27:59,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:01,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:02,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:28:05,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:06,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:10,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:11,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:11,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:14,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:28:15,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:16,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:20,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:22,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:23,146 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:28:26,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:28:26,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:27,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:31,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:32,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:35,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:28:37,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:37,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:42,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:44,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:47,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:47,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:28:49,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:52,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:53,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:28:54,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:28:57,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:28:58,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:28:59,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:03,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:05,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:07,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:29:08,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:13,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:15,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:17,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:29:18,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:20,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:23,148 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:29:24,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:26,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:26,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:27,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:29:27,508 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:29:27,509 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:29:27,512 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:29:28,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:29:28,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:29:29,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:30,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:29:31,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:34,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:37,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:39,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:40,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:29:42,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:44,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:48,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:50,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:52,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:29:53,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:29:53,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:56,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:29:59,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:29:59,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:01,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:02,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:30:04,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:06,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:09,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:11,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:12,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:30:14,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:16,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:19,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:21,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:22,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:30:23,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:30:24,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:27,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:30,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:32,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:34,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:30:35,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:37,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:41,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:42,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:44,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:30:46,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:47,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:51,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:53,149 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:30:53,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:30:54,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:30:56,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:30:58,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:02,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:03,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:31:03,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:07,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:08,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:13,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:13,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:14,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:15,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:31:18,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:19,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:23,151 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:31:24,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:25,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:26,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:31:29,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:30,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:34,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:35,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:38,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:31:39,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:40,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:44,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:44,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:45,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:48,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:31:49,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:50,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:53,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:31:54,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:31:56,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:31:58,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:31:59,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:01,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:06,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:10,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:32:11,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:11,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:16,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:16,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:20,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:32:21,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:21,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:23,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:32:27,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:27,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:30,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:32:32,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:32,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:37,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:38,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:40,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:32:43,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:43,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:48,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:49,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:50,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:32:53,153 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:32:53,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:54,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:32:58,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:32:58,250 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:32:58,250 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:32:58,252 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:32:58,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:32:58,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:32:59,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:00,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:33:03,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:04,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:09,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:10,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:10,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:33:14,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:15,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:19,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:33:20,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:20,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:23,154 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:33:25,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:26,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:31,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:31,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:32,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:33:36,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:36,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:41,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:42,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:33:42,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:46,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:47,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:51,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:52,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:33:53,155 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:33:54,327 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:33:56,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:33:58,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:02,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:03,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:04,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:34:07,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:08,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:12,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:12,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:13,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:16,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:34:17,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:19,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:23,156 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:34:23,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:24,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:26,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:34:28,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:29,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:33,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:34,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:36,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:34:38,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:40,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:43,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:45,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:50,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:50,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:34:52,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:34:53,157 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:34:55,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:34:56,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:00,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:01,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:04,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:35:06,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:06,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:11,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:11,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:16,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:17,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:35:17,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:22,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:23,158 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:35:23,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:26,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:35:27,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:28,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:32,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:33,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:36,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:35:38,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:38,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:43,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:44,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:44,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:46,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:35:48,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:49,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:53,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:35:54,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:54,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:54,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:35:58,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:35:59,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:35:59,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:04,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:06,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:08,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:36:09,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:11,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:14,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:16,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:16,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:20,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:20,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:36:21,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:23,159 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:36:26,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:27,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:30,648 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:36:31,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:32,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:36,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:38,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:40,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:36:40,125 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:36:40,126 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:36:40,130 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:36:40,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:36:40,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:36:42,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:42,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:36:43,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:47,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:49,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:49,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:52,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:36:52,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:36:53,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:36:54,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:36:58,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:00,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:00,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:02,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:37:03,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:05,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:08,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:10,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:10,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:12,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:37:14,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:15,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:19,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:20,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:23,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:37:23,160 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:37:25,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:25,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:30,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:31,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:31,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:35,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:37:35,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:36,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:40,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:41,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:41,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:45,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:37:45,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:46,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:50,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:52,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:37:53,161 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:37:54,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:37:56,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:37:57,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:01,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:02,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:02,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:04,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:38:06,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:11,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:14,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:16,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:38:16,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:19,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:21,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:23,163 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:38:25,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:27,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:28,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:38:30,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:32,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:35,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:35,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:38,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:38,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:38:40,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:43,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:46,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:48,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:48,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:38:51,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:53,164 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:38:54,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:38:56,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:38:59,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:39:00,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:01,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:05,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:08,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:10,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:11,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:39:13,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:15,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:18,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:20,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:21,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:39:23,164 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:39:23,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:26,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:29,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:31,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:33,391 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:39:34,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:36,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:40,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:42,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:39:42,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:45,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:47,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:50,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:52,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:39:53,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:39:53,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:39:55,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:39:58,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:01,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:03,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:04,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:40:06,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:08,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:11,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:12,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:40:12,848 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:40:12,849 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:40:12,851 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:40:13,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:40:13,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:14,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:40:16,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:18,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:21,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:21,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:23,165 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:40:24,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:24,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:40:26,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:29,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:32,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:34,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:34,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:40:37,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:39,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:42,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:44,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:45,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:40:47,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:49,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:52,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:52,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:40:53,166 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:40:55,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:40:55,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:40:57,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:00,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:02,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:05,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:41:05,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:07,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:10,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:13,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:16,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:17,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:41:18,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:21,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:23,168 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:41:23,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:23,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:27,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:27,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:41:28,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:32,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:33,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:37,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:41:37,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:38,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:42,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:44,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:47,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:41:47,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:49,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:52,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:41:53,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:41:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:41:57,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:41:58,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:00,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:03,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:05,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:08,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:10,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:13,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:15,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:18,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:23,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:42:24,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:25,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:29,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:30,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:32,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:42:34,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:35,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:39,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:41,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:44,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:45,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:42:46,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:49,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:52,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:52,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:42:53,170 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:42:55,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:42:55,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:42:57,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:00,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:02,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:05,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:43:05,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:07,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:10,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:13,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:43:13,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:16,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:18,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:21,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:23,171 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:43:24,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:24,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:25,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:43:27,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:29,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:32,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:34,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:35,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:43:37,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:39,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:42,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:44,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:45,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:43:47,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:49,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:52,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:43:53,171 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:43:55,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:43:55,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:43:58,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:01,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:02,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:44:02,744 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:44:02,745 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:44:02,747 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:44:03,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:44:03,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:05,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:44:06,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:08,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:12,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:13,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:44:14,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:17,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:19,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:23,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:23,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:23,172 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:44:24,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:25,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:44:28,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:29,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:33,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:34,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:44:35,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:38,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:40,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:44,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:44,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:44:45,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:49,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:50,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:44:53,174 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:44:55,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:44:55,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:44:56,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:00,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:05,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:45:05,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:06,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:10,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:11,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:15,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:45:16,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:17,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:22,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:22,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:22,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:23,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:45:25,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:45:27,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:27,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:32,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:33,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:35,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:45:38,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:38,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:43,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:44,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:45,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:45:49,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:49,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:53,175 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:45:54,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:54,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:54,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:45:55,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:45:59,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:45:59,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:04,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:05,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:05,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:46:10,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:10,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:15,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:46:15,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:16,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:21,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:22,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:22,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:23,177 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:46:25,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:46:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:27,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:31,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:32,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:35,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:46:37,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:37,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:42,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:43,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:45,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:46:47,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:48,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:52,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:53,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:53,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:46:53,177 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:46:55,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:46:57,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:46:58,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:02,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:03,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:07,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:07,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:08,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:12,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:13,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:13,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:17,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:18,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:18,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:23,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:23,179 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:47:24,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:25,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:47:25,192 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:47:25,193 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:47:25,196 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:47:26,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:47:27,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:28,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:28,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:29,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:33,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:34,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:37,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:38,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:39,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:43,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:44,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:47,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:48,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:50,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:53,181 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:47:54,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:47:55,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:47:57,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:47:59,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:00,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:04,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:05,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:07,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:48:10,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:11,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:15,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:16,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:17,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:48:20,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:21,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:23,181 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:48:26,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:27,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:27,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:48:31,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:33,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:37,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:37,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:48:38,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:42,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:44,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:47,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:47,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:48:49,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:52,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:48:53,183 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:48:55,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:48:55,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:48:57,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:00,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:02,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:05,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:05,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:49:08,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:11,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:13,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:15,941 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:49:16,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:19,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:23,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:23,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:23,196 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:49:24,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:26,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:49:28,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:29,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:33,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:34,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:36,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:49:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:39,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:44,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:45,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:46,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:49:49,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:50,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:53,184 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:49:54,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:49:56,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:49:59,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:01,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:04,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:06,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:07,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:50:09,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:11,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:14,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:17,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:50:17,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:20,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:22,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:23,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:50:26,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:27,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:50:28,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:31,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:33,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:36,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:38,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:41,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:43,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:43,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:50:46,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:48,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:51,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:53,186 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:50:53,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:50:54,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:50:56,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:50:59,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:00,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:51:00,886 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:51:00,887 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:51:00,890 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:51:01,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:51:01,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:51:02,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:04,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:07,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:09,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:11,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:51:13,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:15,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:19,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:21,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:21,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:51:23,187 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:51:24,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:27,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:30,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:30,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:33,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:33,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:51:35,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:38,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:40,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:43,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:44,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:51:45,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:48,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:50,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:50,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:53,187 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:51:54,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:51:54,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:51:55,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:51:59,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:02,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:04,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:06,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:52:07,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:09,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:13,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:15,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:16,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:52:18,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:20,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:23,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:52:23,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:26,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:27,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:52:28,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:31,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:33,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:36,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:38,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:39,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:52:41,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:43,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:46,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:48,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:49,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:52:52,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:53,190 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:52:53,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:53,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:58,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:52:59,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:52:59,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:53:03,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:04,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:08,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:09,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:53:10,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:10,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:14,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:15,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:19,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:20,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:21,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:53:23,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:53:25,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:25,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:30,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:31,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:33,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:53:35,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:36,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:40,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:41,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:44,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:53:45,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:46,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:50,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:52,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:52,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:53:53,191 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:53:54,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:53:56,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:53:57,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:01,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:02,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:04,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:54:06,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:07,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:11,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:12,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:14,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:54:16,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:17,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:21,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:23,193 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:54:23,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:24,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:54:27,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:28,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:32,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:33,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:54:33,746 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:54:33,748 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:54:33,751 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:54:34,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:34,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:54:36,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:54:37,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:39,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:42,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:44,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:45,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:54:48,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:49,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:53,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:54:53,196 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:54:54,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:54,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:54:55,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:54:58,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:00,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:03,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:05,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:55:05,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:08,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:10,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:13,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:15,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:55:16,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:19,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:21,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:23,196 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:55:24,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:25,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:55:26,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:26,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:30,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:31,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:35,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:37,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:37,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:55:40,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:42,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:45,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:47,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:48,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:55:51,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:53,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:55:53,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:55:57,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:55:58,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:02,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:04,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:08,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:08,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:56:09,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:13,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:15,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:18,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:56:18,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:20,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:23,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:56:24,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:26,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:26,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:28,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:56:29,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:31,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:34,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:37,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:40,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:40,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:56:42,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:45,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:47,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:48,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:56:50,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:52,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:53,197 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:56:56,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:56:58,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:58,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:56:58,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:01,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:03,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:07,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:08,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:09,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:12,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:14,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:14,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:17,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:18,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:19,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:22,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:23,199 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:57:25,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:26,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:28,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:30,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:33,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:35,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:37,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:38,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:40,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:43,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:46,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:47,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:49,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:51,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:57:53,200 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:57:55,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:57:55,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:57:57,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:00,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:03,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:03,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:04,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 19:58:04,613 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 19:58:04,614 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 19:58:04,616 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 19:58:05,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 19:58:05,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:06,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:58:08,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:10,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:13,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:13,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:16,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:16,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:58:18,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:21,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:23,201 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:58:24,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:26,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:58:26,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:29,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:31,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:35,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:36,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:58:37,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:40,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:42,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:45,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:46,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:58:48,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:50,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:53,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:58:54,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:58:56,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:58:56,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:58:59,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:02,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:04,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:06,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:59:07,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:09,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:13,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:13,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:15,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:16,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:59:18,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:20,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:23,203 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:59:23,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:25,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:26,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:59:28,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:30,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:34,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:36,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:36,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:59:39,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:41,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:45,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:45,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:46,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:59:47,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:51,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:52,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 19:59:53,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 19:59:54,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 19:59:56,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 19:59:58,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:02,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:03,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:04,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:00:07,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:08,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:12,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:12,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:13,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:14,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:00:17,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:18,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:22,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:23,204 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:00:23,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:00:24,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:27,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:33,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:34,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:00:34,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:38,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:39,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:43,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:44,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:00:45,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:48,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:50,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:53,205 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:00:54,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:00:54,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:00:55,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:00:59,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:00,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:04,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:04,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:05,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:09,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:10,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:14,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:15,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:15,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:16,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:20,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:21,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:23,206 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:01:23,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:01:23,938 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:01:23,939 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:01:23,942 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:01:24,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:01:24,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:25,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:26,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:26,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:30,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:31,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:36,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:36,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:37,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:42,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:42,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:44,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:47,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:48,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:52,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:53,207 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:01:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:01:54,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:01:57,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:01:59,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:03,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:04,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:02:05,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:08,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:10,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:14,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:15,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:16,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:02:19,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:20,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:23,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:02:24,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:02:25,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:26,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:30,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:31,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:35,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:02:36,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:37,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:41,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:42,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:45,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:02:46,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:47,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:51,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:53,208 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:02:53,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:02:54,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:02:56,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:02:58,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:02,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:02,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:03,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:04,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:03:07,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:08,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:12,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:14,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:14,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:03:18,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:19,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:23,209 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:03:23,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:24,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:03:25,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:28,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:30,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:33,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:34,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:03:36,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:38,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:41,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:44,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:44,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:03:46,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:49,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:51,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:03:53,210 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:03:54,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:03:55,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:03:57,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:01,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:02,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:04,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:04:06,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:07,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:11,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:12,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:14,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:04:16,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:18,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:22,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:23,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:04:23,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:25,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:04:27,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:33,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:33,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:35,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:04:38,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:38,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:42,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:04:42,354 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:04:42,355 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:04:42,357 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:04:43,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:04:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:44,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:45,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:04:49,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:49,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:53,211 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:04:54,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:04:55,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:04:55,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:00,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:00,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:00,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:03,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:05,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:05,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:10,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:11,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:12,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:16,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:16,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:21,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:22,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:23,212 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:05:24,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:27,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:32,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:32,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:34,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:37,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:38,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:43,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:43,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:44,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:48,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:48,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:53,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:53,223 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:05:53,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:05:58,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:05:58,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:05:59,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:03,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:04,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:08,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:06:08,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:09,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:13,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:14,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:18,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:06:19,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:20,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:23,213 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:06:25,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:26,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:28,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:06:30,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:31,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:35,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:37,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:38,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:06:41,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:42,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:46,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:47,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:49,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:06:51,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:52,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:53,213 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:06:57,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:06:58,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:06:59,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:07:02,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:03,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:07,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:08,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:09,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:07:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:15,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:15,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:18,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:19,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:07:20,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:23,215 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:07:24,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:25,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:27,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:07:29,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:30,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:34,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:36,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:37,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:07:39,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:41,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:44,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:47,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:49,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:07:50,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:52,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:53,216 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:07:56,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:07:57,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:07:59,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:01,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:03,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:05,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:08:05,992 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:08:05,992 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:08:05,995 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:08:06,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:08:06,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:06,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:08,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:11,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:14,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:14,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:16,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:17,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:19,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:22,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:23,217 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:08:25,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:26,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:28,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:30,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:33,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:35,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:36,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:39,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:40,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:44,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:46,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:46,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:49,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:51,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:53,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:08:54,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:08:56,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:08:56,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:08:59,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:01,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:05,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:07,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:09:07,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:10,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:12,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:15,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:17,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:09:17,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:20,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:23,221 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:09:23,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:25,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:27,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:09:28,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:30,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:33,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:33,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:35,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:37,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:09:38,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:40,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:44,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:45,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:09:46,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:49,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:51,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:53,224 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:09:54,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:09:56,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:09:57,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:09:59,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:01,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:04,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:06,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:07,429 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:10:10,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:11,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:16,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:17,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:17,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:10:21,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:22,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:23,224 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:10:25,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:10:26,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:28,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:32,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:33,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:35,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:10:37,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:38,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:43,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:44,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:45,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:10:48,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:49,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:53,225 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:10:54,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:10:54,789 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:10:55,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:10:59,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:00,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:04,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:04,855 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:11:06,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:09,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:11,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:14,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:11:15,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:16,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:21,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:21,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:23,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:11:23,044 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:11:23,044 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:11:23,046 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:11:23,225 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:11:24,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:11:25,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:11:26,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:27,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:32,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:32,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:32,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:35,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:11:37,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:37,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:42,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:43,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:43,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:11:47,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:48,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:53,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:11:53,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:53,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:11:55,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:11:58,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:11:58,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:03,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:04,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:05,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:12:08,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:11,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:13,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:12:14,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:16,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:19,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:21,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:23,226 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:12:23,422 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:12:24,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:26,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:29,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:32,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:32,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:33,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:12:34,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:37,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:39,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:42,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:43,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:12:45,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:47,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:50,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:53,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:12:53,227 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:12:53,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:12:56,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:12:58,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:01,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:03,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:03,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:03,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:13:06,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:08,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:12,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:13,886 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:13:14,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:17,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:20,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:20,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:23,227 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:13:23,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:23,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:13:25,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:28,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:30,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:33,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:13:33,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:36,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:38,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:41,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:43,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:13:43,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:46,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:48,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:52,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:53,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:13:53,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:13:54,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:13:57,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:13:59,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:02,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:03,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:14:04,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:08,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:09,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:13,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:14:13,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:15,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:19,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:19,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:20,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:23,229 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:14:23,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:14:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:25,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:30,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:30,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:31,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:14:35,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:35,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:40,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:40,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:40,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:14:40,735 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:14:40,736 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:14:40,738 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:14:41,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:14:43,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:14:45,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:45,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:50,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:50,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:14:53,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:14:53,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:14:55,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:14:56,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:00,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:01,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:03,632 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:15:06,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:06,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:11,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:12,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:13,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:15:16,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:17,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:21,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:23,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:23,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:15:23,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:15:26,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:28,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:32,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:33,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:33,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:15:37,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:38,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:42,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:44,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:45,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:15:47,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:49,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:53,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:15:53,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:15:53,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:15:55,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:15:58,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:00,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:04,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:04,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:04,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:16:06,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:09,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:11,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:13,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:16:14,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:17,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:20,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:20,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:22,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:23,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:16:23,230 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:16:25,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:28,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:31,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:33,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:35,320 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:16:36,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:38,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:41,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:44,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:45,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:16:46,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:49,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:52,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:16:53,231 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:16:53,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:16:55,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:16:57,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:00,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:02,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:03,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:17:06,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:07,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:11,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:13,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:17:13,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:16,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:18,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:22,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:23,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:17:23,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:17:24,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:28,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:30,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:33,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:17:34,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:35,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:39,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:40,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:43,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:17:44,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:45,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:49,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:50,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:50,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:17:53,232 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:17:53,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:17:55,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:17:55,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:00,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:00,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:01,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:18:01,092 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:18:01,093 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:18:01,095 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:18:01,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:18:03,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:18:05,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:06,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:11,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:11,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:11,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:18:16,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:16,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:21,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:22,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:18:22,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:23,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:18:27,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:28,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:32,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:33,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:34,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:18:38,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:38,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:43,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:44,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:18:44,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:48,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:49,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:18:53,233 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:18:54,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:18:54,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:18:55,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:00,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:00,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:03,407 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:19:05,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:05,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:10,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:11,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:13,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:19:16,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:16,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:21,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:21,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:23,234 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:19:23,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:19:26,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:27,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:32,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:32,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:33,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:19:37,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:37,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:42,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:43,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:43,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:19:47,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:48,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:53,234 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:19:53,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:19:53,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:19:54,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:19:58,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:00,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:03,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:03,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:20:05,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:08,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:10,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:13,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:13,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:20:15,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:18,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:20,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:23,235 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:20:23,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:20:24,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:29,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:30,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:34,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:35,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:35,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:38,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:20:40,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:40,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:45,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:46,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:48,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:20:50,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:51,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:53,236 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:20:56,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:20:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:20:58,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:21:01,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:02,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:07,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:07,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:08,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:21:12,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:12,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:18,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:18,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:21:18,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:23,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:23,239 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:21:24,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:25,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:21:25,594 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:21:25,594 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:21:25,597 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:21:26,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:21:28,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:21:28,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:29,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:33,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:35,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:35,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:38,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:21:39,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:40,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:44,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:46,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:48,558 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:21:50,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:51,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:53,240 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:21:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:21:56,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:21:57,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:01,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:06,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:07,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:07,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:11,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:12,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:17,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:17,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:17,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:22,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:23,241 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:22:24,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:27,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:28,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:29,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:33,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:34,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:34,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:35,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:38,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:39,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:43,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:45,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:47,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:49,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:50,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:53,243 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:22:54,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:22:55,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:22:58,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:22:59,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:00,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:04,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:06,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:08,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:23:10,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:11,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:16,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:23:16,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:16,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:21,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:22,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:23,243 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:23:26,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:23:27,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:27,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:32,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:34,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:37,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:38,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:23:39,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:42,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:45,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:46,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:23:48,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:50,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:53,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:23:53,246 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:23:55,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:23:56,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:23:58,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:00,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:06,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:06,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:06,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:24:09,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:11,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:14,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:16,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:16,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:24:19,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:21,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:23,247 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:24:25,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:26,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:26,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:24:30,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:31,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:36,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:36,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:24:37,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:41,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:42,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:45,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:24:45,120 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:24:45,121 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:24:45,123 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:24:45,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:24:45,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:24:47,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:47,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:53,249 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:24:53,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:24:55,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:24:57,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:24:58,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:02,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:04,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:05,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:25:08,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:09,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:13,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:14,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:16,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:25:19,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:19,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:23,250 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:25:24,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:25,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:26,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:25:29,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:30,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:35,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:35,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:36,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:25:40,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:40,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:46,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:46,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:25:46,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:51,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:51,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:25:53,251 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:25:56,372 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:25:56,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:25:57,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:01,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:03,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:06,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:26:07,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:08,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:12,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:13,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:16,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:26:17,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:22,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:23,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:26:24,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:26,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:26:28,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:29,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:33,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:35,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:36,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:26:39,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:40,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:44,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:45,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:46,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:26:49,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:52,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:52,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:52,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:26:53,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:26:54,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:26:55,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:26:57,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:00,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:02,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:04,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:27:06,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:07,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:11,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:13,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:14,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:27:16,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:18,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:21,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:23,253 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:27:24,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:25,002 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:27:27,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:29,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:32,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:34,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:27:34,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:37,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:39,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:42,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:44,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:27:44,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:47,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:49,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:52,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:27:53,254 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:27:54,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:27:55,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:27:58,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:01,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:02,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:28:02,925 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:28:02,926 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:28:02,928 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:28:03,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:28:03,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:04,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:28:06,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:08,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:12,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:14,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:16,416 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:28:17,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:19,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:22,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:22,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:22,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:23,255 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:28:24,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:28:25,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:27,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:30,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:33,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:34,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:28:36,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:38,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:41,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:44,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:46,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:28:47,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:49,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:52,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:53,256 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:28:54,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:28:56,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:28:58,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:28:59,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:03,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:05,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:06,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:29:09,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:10,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:14,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:16,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:16,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:29:19,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:22,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:23,257 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:29:24,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:29:25,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:27,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:30,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:33,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:34,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:29:36,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:38,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:41,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:43,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:45,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:29:46,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:48,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:51,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:53,268 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:29:54,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:29:55,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:29:57,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:29:59,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:02,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:04,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:05,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:30:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:09,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:12,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:14,278 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:30:14,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:18,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:19,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:23,258 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:30:23,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:24,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:30:25,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:28,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:30,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:34,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:30:34,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:35,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:39,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:40,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:45,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:45,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:46,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:30:50,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:50,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:53,259 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:30:55,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:30:56,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:30:56,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:00,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:01,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:05,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:06,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:06,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:10,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:12,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:14,750 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:15,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:17,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:20,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:23,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:23,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:23,259 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:31:23,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:31:23,322 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:31:23,323 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:31:23,326 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:31:23,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:31:24,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:26,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:28,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:31,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:34,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:34,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:37,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:39,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:42,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:44,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:44,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:48,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:53,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:31:53,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:31:55,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:31:57,051 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:31:58,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:00,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:03,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:05,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:07,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:32:09,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:10,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:15,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:16,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:17,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:32:20,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:21,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:23,261 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:32:25,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:26,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:27,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:32:30,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:31,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:35,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:36,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:37,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:32:40,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:43,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:46,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:32:46,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:48,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:51,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:53,264 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:32:53,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:53,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:32:56,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:32:57,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:32:58,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:02,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:03,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:06,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:33:07,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:08,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:12,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:13,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:16,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:33:18,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:18,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:23,265 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:33:24,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:24,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:24,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:26,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:33:29,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:29,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:34,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:34,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:34,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:36,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:33:39,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:40,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:45,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:46,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:46,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:33:50,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:51,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:53,267 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:33:55,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:33:56,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:33:57,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:00,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:01,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:05,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:07,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:07,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:11,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:12,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:16,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:17,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:18,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:21,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:23,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:23,268 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:34:27,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:27,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:28,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:32,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:33,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:37,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:37,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:39,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:42,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:44,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:46,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:34:46,383 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:34:46,384 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:34:46,491 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:34:47,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:34:48,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:48,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:49,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:50,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:53,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:34:53,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:34:56,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:34:57,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:34:58,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:01,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:05,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:05,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:06,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:09,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:35:10,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:11,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:15,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:16,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:20,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:21,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:23,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:35:23,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:35:25,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:27,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:31,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:32,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:33,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:35:36,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:37,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:40,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:35:42,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:43,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:47,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:48,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:50,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:35:52,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:53,269 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:35:54,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:35:58,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:35:59,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:00,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:36:03,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:04,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:09,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:09,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:09,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:10,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:36:14,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:15,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:19,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:20,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:21,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:36:23,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:36:24,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:26,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:30,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:31,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:36:31,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:35,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:36,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:40,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:41,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:36:41,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:45,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:46,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:51,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:51,313 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:36:52,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:36:53,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:36:56,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:36:58,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:01,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:37:01,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:04,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:08,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:09,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:11,461 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:37:13,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:14,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:18,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:20,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:21,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:37:23,270 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:37:23,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:25,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:29,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:29,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:37:30,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:34,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:35,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:39,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:37:39,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:41,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:45,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:46,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:49,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:37:50,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:52,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:53,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:37:56,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:37:57,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:37:59,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:01,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:02,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:07,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:07,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:07,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:08,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:38:08,590 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:38:08,591 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:38:08,594 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:38:08,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:38:08,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:10,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:12,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:13,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:17,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:18,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:19,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:22,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:23,271 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:38:25,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:28,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:29,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:31,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:33,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:36,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:38,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:39,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:41,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:43,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:46,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:48,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:49,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:38:52,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:53,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:38:53,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:38:57,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:38:59,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:01,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:39:02,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:04,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:09,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:39:09,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:13,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:16,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:19,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:19,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:39:21,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:23,272 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:39:24,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:27,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:29,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:39:29,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:32,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:34,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:38,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:38,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:39,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:39:40,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:43,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:45,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:49,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:49,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:39:50,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:53,274 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:39:54,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:39:56,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:39:59,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:40:00,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:01,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:05,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:06,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:09,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:40:10,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:12,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:16,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:17,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:19,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:40:21,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:22,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:23,274 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:40:27,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:28,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:31,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:40:32,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:33,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:38,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:39,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:40:39,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:43,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:44,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:49,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:40:49,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:50,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:53,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:40:54,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:40:56,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:40:59,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:41:00,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:01,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:05,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:06,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:10,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:10,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:11,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:41:11,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:16,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:17,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:19,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:41:22,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:22,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:23,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:41:27,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:28,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:28,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:41:28,106 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:41:28,107 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:41:28,110 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:41:28,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:41:29,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:41:32,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:33,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:38,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:38,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:39,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:41:43,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:43,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:48,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:49,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:49,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:41:53,275 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:41:53,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:54,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:41:59,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:41:59,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:42:00,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:05,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:05,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:07,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:42:10,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:10,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:15,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:16,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:17,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:42:21,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:21,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:23,277 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:42:26,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:26,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:27,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:42:31,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:31,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:36,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:36,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:38,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:42:41,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:41,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:46,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:46,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:48,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:42:51,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:51,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:53,278 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:42:57,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:57,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:42:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:42:58,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:43:02,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:02,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:07,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:43:07,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:07,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:12,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:12,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:17,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:43:18,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:18,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:23,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:23,280 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:43:24,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:27,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:43:29,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:29,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:34,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:34,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:37,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:43:39,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:39,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:44,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:45,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:47,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:43:50,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:50,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:53,281 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:43:55,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:43:55,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:55,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:43:57,613 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:00,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:00,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:05,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:06,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:06,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:11,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:11,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:15,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:16,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:16,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:21,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:21,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:23,282 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:44:25,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:27,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:27,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:32,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:32,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:35,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:37,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:38,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:43,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:44,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:44:45,342 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:44:45,343 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:44:45,345 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:44:45,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:44:47,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:48,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:49,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:53,283 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:44:54,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:44:54,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:56,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:44:59,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:44:59,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:05,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:05,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:06,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:45:10,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:10,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:16,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:45:16,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:16,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:21,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:21,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:23,285 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:45:26,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:45:26,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:27,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:32,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:32,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:36,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:45:37,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:37,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:42,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:43,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:43,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:46,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:45:48,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:48,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:53,298 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:45:53,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:54,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:45:56,423 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:45:59,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:45:59,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:04,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:05,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:05,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:46:09,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:10,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:14,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:15,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:46:16,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:19,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:21,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:23,286 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:46:25,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:25,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:46:26,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:30,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:31,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:35,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:46:36,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:37,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:42,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:42,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:42,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:45,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:46:47,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:47,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:52,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:46:53,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:46:53,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:55,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:46:58,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:46:58,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:03,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:47:04,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:04,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:09,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:09,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:13,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:47:14,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:15,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:19,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:20,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:23,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:47:24,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:47:25,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:25,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:30,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:31,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:34,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:47:36,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:36,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:41,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:42,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:42,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:44,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:47:46,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:47,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:51,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:53,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:47:53,287 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:47:54,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:47:57,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:47:58,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:01,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:48:01,596 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:48:01,597 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:48:01,600 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:48:02,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:48:02,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:03,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:03,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:04,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:07,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:09,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:09,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:12,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:13,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:14,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:18,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:20,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:22,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:23,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:23,288 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:48:25,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:30,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:32,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:33,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:35,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:38,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:40,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:41,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:42,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:44,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:46,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:49,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:51,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:51,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:48:53,289 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:48:54,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:48:56,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:48:59,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:01,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:49:01,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:05,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:06,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:10,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:11,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:49:12,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:15,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:17,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:20,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:21,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:49:23,289 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:49:23,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:25,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:29,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:29,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:29,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:30,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:32,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:49:34,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:35,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:40,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:41,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:42,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:49:45,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:46,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:50,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:52,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:49:52,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:49:53,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:49:55,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:49:57,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:01,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:02,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:06,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:50:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:07,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:11,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:12,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:16,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:17,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:21,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:22,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:23,290 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:50:26,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:31,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:33,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:36,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:38,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:41,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:43,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:46,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:48,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:51,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:53,291 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:50:54,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:50:56,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:50:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:01,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:04,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:06,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:12,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:14,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:51:14,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:17,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:19,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:22,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:23,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:51:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:26,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:51:27,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:30,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:32,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:35,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:36,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:51:37,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:40,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:42,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:43,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:51:46,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:47,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:51,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:53,292 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:51:53,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:51:53,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:51:57,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:51:58,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:02,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:04,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:04,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:52:08,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:09,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:13,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:14,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:16,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:52:19,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:19,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:23,295 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:52:23,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:52:23,888 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:52:23,889 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:52:23,894 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:52:24,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:52:24,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:52:24,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:25,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:26,203 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:52:29,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:30,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:34,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:35,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:39,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:40,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:44,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:52:45,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:45,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:50,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:52:51,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:53,294 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:52:54,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:52:56,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:52:56,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:01,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:04,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:53:06,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:07,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:12,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:12,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:16,657 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:53:17,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:17,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:22,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:22,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:23,295 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:53:26,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:53:27,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:27,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:32,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:32,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:36,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:53:37,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:38,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:43,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:43,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:43,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:46,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:53:48,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:49,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:53,297 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:53:54,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:54,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:53:56,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:53:59,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:53:59,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:04,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:04,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:09,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:54:09,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:10,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:14,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:14,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:14,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:15,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:18,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:54:19,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:20,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:23,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:54:24,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:26,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:26,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:54:29,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:31,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:35,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:36,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:38,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:54:40,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:41,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:46,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:46,299 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:54:47,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:51,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:52,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:54:53,299 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:54:56,364 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:54:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:54:57,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:01,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:02,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:06,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:07,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:08,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:12,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:13,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:16,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:17,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:19,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:22,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:23,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:55:25,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:26,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:27,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:30,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:33,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:35,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:36,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:38,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:40,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:43,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:45,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:46,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:48,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:50,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:53,300 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:55:53,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:55:53,309 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:55:53,309 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:55:53,311 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:55:53,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:55:53,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:54,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:55:54,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:55:56,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:55:59,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:01,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:04,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:04,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:56:07,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:09,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:12,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:15,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:56:15,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:18,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:21,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:23,301 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:56:23,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:25,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:56:26,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:28,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:32,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:33,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:37,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:56:37,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:38,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:43,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:44,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:45,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:56:48,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:49,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:53,302 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:56:53,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:56:54,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:56:55,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:56:58,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:00,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:04,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:57:04,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:05,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:09,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:10,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:14,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:57:15,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:16,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:21,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:21,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:23,302 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:57:24,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:57:26,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:27,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:32,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:32,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:32,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:34,581 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:57:37,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:37,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:42,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:42,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:44,649 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:57:47,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:48,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:52,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:53,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:57:53,303 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:57:54,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:57:57,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:57:58,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:03,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:03,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:04,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:58:08,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:08,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:14,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:14,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:14,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:58:19,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:19,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:22,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:58:23,304 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:58:25,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:25,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:30,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:30,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:30,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:30,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:33,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:58:35,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:36,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:41,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:41,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:45,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:58:46,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:46,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:51,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:58:52,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:53,304 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:58:55,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:58:57,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:58:57,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:02,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:03,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:03,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:05,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:07,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:08,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:12,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:13,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:14,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 20:59:14,355 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 20:59:14,356 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 20:59:14,359 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 20:59:14,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 20:59:16,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:17,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:18,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:18,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:23,305 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:59:24,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:24,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:26,485 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:29,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:30,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:34,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:34,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:35,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:36,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:39,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:41,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:45,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:46,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:46,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:50,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:51,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 20:59:53,306 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 20:59:55,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 20:59:56,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 20:59:56,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:00,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:01,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:06,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:06,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:00:07,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:11,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:12,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:17,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:17,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:18,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:00:22,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:22,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:23,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:00:27,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:28,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:28,989 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:00:32,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:33,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:38,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:39,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:00:39,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:43,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:44,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:48,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:49,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:00:49,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:53,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:00:53,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:55,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:00:58,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:00:59,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:01:00,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:03,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:05,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:09,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:01:09,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:11,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:15,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:16,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:19,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:01:20,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:22,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:23,307 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:01:26,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:28,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:29,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:01:31,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:33,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:37,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:38,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:39,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:01:42,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:43,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:48,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:49,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:01:49,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:53,308 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:01:53,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:55,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:01:58,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:01:59,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:00,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:03,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:05,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:08,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:09,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:10,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:13,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:15,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:18,732 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:18,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:21,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:23,308 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:02:23,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:26,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:28,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:29,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:31,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:35,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:36,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:02:36,265 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:02:36,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:02:36,268 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:02:36,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:02:36,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:37,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:38,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:40,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:42,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:45,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:46,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:47,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:50,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:52,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:02:53,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:02:56,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:02:57,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:02:58,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:01,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:03,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:08,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:09,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:03:11,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:13,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:17,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:17,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:19,211 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:03:19,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:22,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:23,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:03:25,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:27,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:29,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:03:31,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:32,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:36,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:38,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:39,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:03:41,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:43,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:47,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:03:47,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:49,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:52,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:53,310 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:03:54,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:03:58,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:03:59,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:01,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:04:04,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:04,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:09,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:10,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:11,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:04:14,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:15,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:19,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:21,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:04:21,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:23,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:04:24,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:28,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:30,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:31,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:04:33,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:38,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:40,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:41,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:04:43,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:45,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:49,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:49,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:51,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:51,881 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:04:53,311 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:04:54,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:04:57,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:04:59,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:00,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:05:03,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:04,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:08,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:10,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:11,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:05:13,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:15,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:18,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:19,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:05:21,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:23,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:05:23,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:27,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:28,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:29,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:05:32,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:33,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:38,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:41,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:05:43,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:44,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:49,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:49,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:50,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:51,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:05:53,312 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:05:54,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:05:56,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:05:59,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:01,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:04,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:06:04,209 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:06:04,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:06:04,211 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:06:04,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:06:04,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:05,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:06,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:07,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:09,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:12,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:14,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:15,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:17,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:19,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:22,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:23,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:06:25,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:25,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:27,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:30,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:32,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:35,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:37,860 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:37,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:40,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:42,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:45,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:47,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:48,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:50,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:53,314 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:06:54,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:06:56,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:06:57,994 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:06:59,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:01,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:04,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:06,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:07,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:07:10,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:13,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:15,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:07:15,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:18,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:20,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:23,315 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:07:24,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:24,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:25,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:07:26,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:29,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:31,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:34,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:34,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:37,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:37,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:07:39,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:42,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:44,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:47,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:47,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:07:49,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:52,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:07:53,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:07:55,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:07:55,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:07:57,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:00,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:02,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:05,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:08:05,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:08,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:11,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:13,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:15,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:08:16,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:19,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:22,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:22,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:23,316 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:08:24,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:25,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:08:27,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:29,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:32,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:34,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:35,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:08:37,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:37,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:39,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:43,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:45,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:45,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:08:48,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:50,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:53,317 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:08:54,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:08:55,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:08:55,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:08:59,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:00,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:04,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:05,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:06,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:11,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:11,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:13,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:16,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:16,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:22,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:22,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:22,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:09:22,632 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:09:22,633 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:09:22,635 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:09:23,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:09:23,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:09:24,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:25,029 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:27,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:27,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:32,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:33,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:36,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:37,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:38,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:42,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:43,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:44,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:48,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:48,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:53,318 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:09:53,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:54,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:09:54,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:54,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:09:58,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:09:59,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:03,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:04,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:10:04,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:08,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:11,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:13,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:10:13,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:16,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:18,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:21,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:23,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:10:23,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:10:24,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:26,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:29,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:32,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:33,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:10:34,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:37,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:39,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:43,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:43,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:10:45,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:48,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:50,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:10:53,326 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:10:53,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:10:54,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:54,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:10:56,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:00,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:01,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:03,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:11:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:06,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:11,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:12,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:13,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:11:16,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:17,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:22,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:22,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:23,327 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:11:23,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:23,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:11:27,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:28,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:32,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:33,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:33,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:11:37,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:38,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:43,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:43,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:11:44,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:48,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:49,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:53,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:11:53,327 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:11:54,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:11:55,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:11:58,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:00,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:04,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:04,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:05,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:10,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:10,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:10,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:14,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:15,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:15,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:20,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:21,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:23,329 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:12:24,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:26,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:26,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:31,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:32,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:34,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:36,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:37,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:41,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:12:41,503 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:12:41,503 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:12:41,506 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:12:42,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:42,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:12:42,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:43,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:44,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:47,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:51,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:12:52,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:53,329 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:12:53,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:12:57,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:12:58,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:01,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:13:03,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:03,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:08,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:10,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:10,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:11,669 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:13:14,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:15,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:19,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:20,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:21,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:13:23,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:13:25,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:25,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:30,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:31,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:31,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:13:35,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:36,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:41,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:41,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:13:42,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:46,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:47,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:51,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:13:52,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:53,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:13:53,330 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:13:58,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:13:59,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:02,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:14:03,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:04,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:08,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:10,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:10,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:12,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:14:14,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:15,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:19,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:21,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:22,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:14:23,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:14:25,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:26,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:31,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:32,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:32,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:14:36,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:37,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:41,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:42,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:14:42,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:46,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:47,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:51,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:52,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:14:53,331 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:14:54,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:14:57,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:14:57,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:02,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:15:03,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:03,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:08,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:09,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:09,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:12,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:15:13,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:14,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:18,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:20,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:22,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:15:23,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:15:24,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:25,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:29,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:31,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:33,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:15:34,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:36,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:39,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:41,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:43,775 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:15:45,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:46,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:50,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:52,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:15:53,332 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:15:53,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:15:55,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:15:57,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:00,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:01,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:16:01,428 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:16:01,428 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:16:01,431 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:16:01,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:16:03,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:03,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:16:05,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:08,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:10,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:13,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:16:14,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:16,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:20,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:21,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:23,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:16:24,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:16:25,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:27,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:30,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:32,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:34,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:16:35,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:37,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:41,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:43,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:44,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:16:46,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:48,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:52,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:53,333 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:16:54,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:16:54,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:16:57,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:16:59,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:03,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:04,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:17:05,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:08,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:10,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:13,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:14,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:17:16,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:18,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:21,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:23,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:17:23,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:24,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:17:26,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:28,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:31,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:34,554 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:17:34,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:36,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:39,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:41,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:44,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:17:45,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:47,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:51,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:53,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:17:53,334 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:17:54,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:17:56,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:17:58,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:01,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:03,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:04,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:18:06,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:08,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:12,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:12,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:18:14,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:17,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:19,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:21,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:18:22,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:23,346 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:18:24,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:27,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:29,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:33,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:34,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:18:34,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:38,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:39,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:42,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:18:43,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:45,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:49,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:50,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:18:52,138 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:18:53,335 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:18:54,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:18:55,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:00,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:00,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:02,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:05,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:05,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:10,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:11,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:12,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:15,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:16,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:20,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:19:20,129 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:19:20,130 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:19:20,132 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:19:20,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:19:21,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:21,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:21,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:22,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:23,336 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:19:26,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:26,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:31,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:32,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:32,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:36,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:37,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:42,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:42,521 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:42,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:47,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:47,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:52,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:52,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:19:53,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:19:53,336 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:19:57,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:19:59,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:02,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:20:03,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:04,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:08,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:10,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:10,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:12,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:20:13,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:15,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:18,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:21,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:22,812 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:20:23,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:20:24,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:26,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:29,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:31,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:34,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:36,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:36,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:38,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:20:39,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:41,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:44,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:47,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:48,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:20:50,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:52,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:53,337 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:20:56,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:20:58,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:20:59,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:01,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:03,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:06,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:08,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:09,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:11,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:14,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:16,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:19,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:20,204 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:21,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:23,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:21:24,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:27,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:30,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:30,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:32,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:36,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:38,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:38,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:41,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:46,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:48,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:49,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:51,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:53,338 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:21:54,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:21:57,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:21:58,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:21:59,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:02,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:04,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:08,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:08,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:10,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:13,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:15,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:18,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:18,701 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:20,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:23,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:22:25,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:25,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:26,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:28,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:30,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:31,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:35,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:37,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:38,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:40,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:42,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:46,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:22:46,004 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:22:46,005 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:22:46,007 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:22:46,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:46,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:22:47,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:48,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:48,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:51,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:53,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:22:53,341 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:22:56,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:22:57,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:22:58,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:02,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:03,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:08,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:09,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:23:09,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:13,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:14,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:18,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:19,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:23:20,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:23,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:23:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:25,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:27,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:23:29,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:30,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:36,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:37,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:23:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:41,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:45,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:46,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:47,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:23:50,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:51,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:23:53,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:23:55,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:23:57,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:00,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:02,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:05,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:07,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:10,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:12,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:15,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:17,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:20,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:22,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:23,343 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:24:25,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:28,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:30,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:32,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:24:33,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:35,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:38,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:41,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:43,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:44,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:24:46,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:48,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:51,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:53,344 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:24:54,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:24:54,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:24:56,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:24:59,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:02,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:04,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:05,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:25:07,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:09,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:12,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:14,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:15,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:25:17,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:19,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:23,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:23,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:23,344 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:25:25,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:25:25,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:28,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:30,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:34,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:35,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:25:36,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:39,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:41,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:44,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:45,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:25:47,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:51,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:52,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:25:53,345 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:25:55,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:25:56,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:25:57,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:01,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:02,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:05,420 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:26:06,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:08,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:13,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:17,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:26:18,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:18,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:23,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:26:24,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:24,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:24,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:26,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:26:29,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:30,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:34,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:35,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:36,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:26:39,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:40,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:44,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:45,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:48,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:26:48,057 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:26:48,058 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:26:48,059 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:26:48,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:26:48,735 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:26:49,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:50,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:26:51,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:26:53,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:26:54,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:26:56,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:00,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:00,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:27:01,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:05,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:06,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:10,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:10,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:10,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:27:11,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:15,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:16,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:20,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:20,949 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:27:22,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:23,347 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:27:25,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:27,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:31,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:27:31,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:33,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:36,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:38,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:41,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:41,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:44,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:45,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:27:46,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:49,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:52,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:53,348 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:27:54,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:27:55,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:27:57,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:27:59,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:02,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:04,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:05,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:28:07,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:09,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:13,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:14,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:15,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:28:18,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:19,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:23,349 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:28:23,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:23,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:24,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:25,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:28:28,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:29,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:34,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:35,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:35,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:28:39,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:40,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:44,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:45,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:47,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:28:49,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:50,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:53,350 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:28:55,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:28:56,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:28:59,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:29:00,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:01,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:05,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:06,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:08,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:29:10,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:12,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:15,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:17,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:18,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:29:20,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:22,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:23,350 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:29:26,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:26,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:27,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:28,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:29:31,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:32,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:36,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:37,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:38,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:29:41,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:42,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:46,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:47,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:51,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:52,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:29:53,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:29:55,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:29:56,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:29:58,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:02,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:03,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:05,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:30:07,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:08,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:13,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:30:14,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:18,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:19,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:22,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:30:22,176 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:30:22,177 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:30:22,180 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:30:22,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:30:23,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:30:23,351 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:30:23,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:24,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:30:24,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:28,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:29,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:34,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:34,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:35,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:30:39,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:39,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:44,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:45,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:45,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:30:49,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:50,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:53,352 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:30:55,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:30:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:30:57,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:31:00,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:00,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:05,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:06,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:07,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:31:10,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:11,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:15,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:16,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:17,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:31:20,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:21,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:23,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:31:26,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:27,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:27,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:31:31,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:32,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:36,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:37,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:41,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:31:42,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:42,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:47,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:47,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:51,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:31:52,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:53,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:31:53,354 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:31:58,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:31:58,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:00,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:32:03,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:03,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:08,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:08,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:11,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:32:13,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:14,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:18,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:20,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:21,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:32:23,355 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:32:23,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:25,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:29,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:30,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:33,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:32:34,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:35,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:40,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:41,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:32:41,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:45,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:46,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:51,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:51,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:32:52,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:32:53,355 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:32:56,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:32:58,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:01,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:01,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:04,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:07,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:09,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:11,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:12,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:12,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:14,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:17,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:19,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:23,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:23,357 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:33:23,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:25,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:28,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:30,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:33,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:35,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:35,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:38,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:40,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:44,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:45,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:46,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:49,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:51,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:53,358 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:33:53,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:33:53,811 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:33:53,812 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:33:53,815 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:33:53,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:33:54,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:54,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:33:54,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:55,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:33:56,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:33:59,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:01,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:05,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:05,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:34:07,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:10,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:12,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:16,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:17,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:34:18,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:21,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:23,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:23,360 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:34:26,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:28,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:34:28,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:31,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:33,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:36,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:34:37,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:39,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:43,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:45,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:34:45,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:48,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:50,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:53,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:34:53,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:34:55,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:34:57,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:34:58,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:00,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:03,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:06,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:07,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:35:08,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:11,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:15,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:17,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:17,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:35:20,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:22,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:23,362 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:35:25,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:25,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:27,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:35:27,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:30,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:32,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:36,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:37,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:35:38,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:41,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:43,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:46,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:47,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:35:48,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:51,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:53,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:35:54,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:35:57,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:35:57,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:35:59,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:02,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:04,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:07,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:36:08,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:10,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:13,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:15,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:19,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:19,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:36:21,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:23,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:36:24,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:27,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:29,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:29,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:29,947 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:36:32,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:34,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:37,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:39,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:43,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:44,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:36:44,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:48,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:53,363 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:36:53,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:36:54,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:36:56,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:36:58,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:01,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:04,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:04,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:37:06,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:09,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:12,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:14,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:37:14,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:17,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:19,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:23,365 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:37:23,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:23,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:37:23,883 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:37:23,884 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:37:23,886 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:37:24,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:37:24,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:25,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:37:28,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:29,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:34,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:35,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:37:35,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:39,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:40,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:45,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:45,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:37:46,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:50,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:51,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:37:53,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:37:55,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:37:55,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:37:57,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:00,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:02,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:05,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:05,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:38:07,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:10,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:12,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:15,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:15,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:15,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:38:17,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:20,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:22,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:23,366 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:38:25,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:38:26,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:28,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:31,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:33,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:36,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:37,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:38:39,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:41,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:44,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:47,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:47,982 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:38:49,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:52,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:53,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:38:55,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:38:57,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:38:58,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:39:00,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:02,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:06,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:08,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:39:08,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:11,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:14,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:14,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:14,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:17,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:18,182 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:39:19,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:22,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:23,368 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:39:24,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:26,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:39:28,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:29,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:33,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:34,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:38,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:42,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:39:43,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:45,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:48,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:51,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:53,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:39:53,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:39:54,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:39:56,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:39:58,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:01,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:01,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:04,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:04,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:06,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:09,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:12,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:13,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:14,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:17,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:19,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:22,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:23,369 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:40:23,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:25,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:27,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:30,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:32,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:32,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:35,740 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:36,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:37,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:41,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:43,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:45,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:46,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:48,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:51,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:53,371 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:40:53,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:40:54,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:40:54,915 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:40:54,916 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:40:54,918 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:40:55,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:40:55,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:56,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:40:56,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:40:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:01,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:04,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:04,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:05,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:41:06,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:09,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:11,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:14,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:16,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:41:17,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:19,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:22,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:23,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:41:25,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:26,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:41:27,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:30,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:32,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:37,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:41,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:42,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:44,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:41:46,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:47,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:51,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:52,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:41:53,372 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:41:54,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:41:56,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:41:58,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:01,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:01,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:04,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:42:06,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:09,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:12,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:14,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:16,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:42:17,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:19,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:22,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:23,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:42:24,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:26,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:42:27,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:29,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:33,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:33,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:35,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:36,688 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:42:38,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:40,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:43,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:45,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:42:45,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:48,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:50,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:53,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:42:53,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:42:56,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:42:58,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:01,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:03,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:43:04,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:04,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:06,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:09,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:13,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:43:15,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:17,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:21,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:21,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:22,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:23,373 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:43:24,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:43:26,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:28,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:31,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:31,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:31,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:34,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:34,130 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:43:36,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:39,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:42,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:44,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:46,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:43:47,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:49,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:52,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:53,375 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:43:54,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:43:56,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:43:57,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:43:59,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:02,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:04,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:07,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:12,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:44:12,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:15,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:17,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:20,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:23,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:23,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:44:24,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:44:25,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:28,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:30,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:33,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:34,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:44:35,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:39,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:40,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:43,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:44:43,079 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:44:43,080 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:44:43,083 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:44:43,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:44:44,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:44,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:44:45,832 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:44:46,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:49,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:51,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:53,376 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:44:53,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:44:54,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:44:56,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:44:59,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:01,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:05,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:06,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:45:06,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:10,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:11,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:15,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:16,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:45:16,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:20,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:21,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:23,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:45:25,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:26,282 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:45:27,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:30,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:32,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:36,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:37,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:38,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:45:41,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:42,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:47,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:47,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:50,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:45:52,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:52,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:45:53,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:45:58,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:45:58,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:00,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:46:03,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:03,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:08,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:08,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:08,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:10,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:46:13,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:14,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:19,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:19,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:20,700 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:46:23,378 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:46:24,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:24,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:29,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:29,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:30,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:46:34,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:34,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:39,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:39,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:44,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:46,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:46:49,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:50,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:53,381 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:46:55,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:46:55,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:46:58,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:47:00,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:01,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:06,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:06,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:06,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:09,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:47:11,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:16,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:16,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:18,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:47:21,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:22,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:23,380 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:47:27,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:27,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:28,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:47:32,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:32,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:37,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:38,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:38,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:40,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:47:43,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:43,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:48,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:48,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:50,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:47:53,391 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:47:53,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:47:53,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:47:58,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:00,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:04,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:04,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:05,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:09,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:11,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:14,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:14,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:16,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:19,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:22,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:22,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:22,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:23,381 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:48:23,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:48:23,774 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:48:23,775 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:48:23,778 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:48:23,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:48:24,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:25,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:25,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:27,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:30,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:32,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:36,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:36,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:37,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:41,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:42,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:46,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:48,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:49,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:51,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:53,382 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:48:53,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:48:56,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:48:59,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:48:59,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:01,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:06,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:06,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:08,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:49:11,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:12,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:16,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:17,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:18,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:49:21,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:22,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:23,382 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:49:27,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:28,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:30,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:49:32,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:33,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:38,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:39,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:39,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:42,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:49:43,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:44,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:48,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:50,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:53,387 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:49:54,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:49:54,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:49:55,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:49:59,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:00,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:04,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:04,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:50:06,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:09,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:11,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:14,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:16,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:17,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:50:20,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:21,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:23,389 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:50:25,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:26,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:26,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:29,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:50:30,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:31,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:35,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:37,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:37,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:37,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:40,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:41,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:50:42,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:50,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:52,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:50:53,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:50:53,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:50:56,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:50:58,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:01,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:03,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:06,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:08,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:11,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:12,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:51:13,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:17,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:18,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:22,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:23,390 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:51:23,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:24,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:51:27,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:28,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:32,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:34,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:36,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:51:38,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:39,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:43,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:45,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:48,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:49,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:51:50,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:53,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:51:54,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:51:55,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:55,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:51:59,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:00,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:00,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:05,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:06,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:09,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:10,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:11,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:15,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:16,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:18,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:52:18,167 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:52:18,167 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:52:18,170 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:52:18,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:52:19,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:20,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:21,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:21,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:23,391 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:52:26,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:27,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:31,393 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:31,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:32,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:36,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:37,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:41,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:42,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:43,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:46,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:47,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:51,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:52,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:52:53,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:52:53,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:52:57,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:52:58,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:02,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:03,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:07,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:08,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:08,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:53:13,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:13,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:18,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:18,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:20,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:53:23,392 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:53:23,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:23,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:28,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:28,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:30,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:53:33,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:33,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:38,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:39,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:39,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:43,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:53:44,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:44,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:49,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:53:49,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:53,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:53:53,393 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:53:54,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:53:55,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:00,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:00,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:03,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:54:05,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:06,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:10,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:10,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:11,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:13,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:54:15,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:16,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:20,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:21,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:23,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:54:23,393 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:54:25,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:27,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:32,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:32,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:35,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:54:37,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:37,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:42,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:42,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:45,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:54:47,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:48,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:53,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:54:53,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:53,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:53,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:53,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:54:57,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:54:58,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:54:58,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:03,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:03,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:07,708 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:55:08,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:09,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:14,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:14,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:16,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:55:19,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:20,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:23,395 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:55:24,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:25,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:28,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:55:29,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:31,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:34,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:36,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:38,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:55:40,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:41,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:45,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:46,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:50,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:51,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:55:51,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:53,396 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:55:55,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:55:57,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:55:59,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:55:59,081 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:55:59,081 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:55:59,086 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:55:59,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:56:00,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:00,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:01,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:03,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:05,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:08,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:11,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:11,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:13,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:16,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:18,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:22,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:23,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:23,396 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:56:24,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:27,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:29,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:32,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:33,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:34,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:37,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:39,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:42,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:45,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:45,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:47,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:50,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:53,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:56:53,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:56:55,675 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:56:55,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:56:58,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:00,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:03,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:05,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:57:06,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:08,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:11,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:15,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:16,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:17,857 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:57:20,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:21,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:23,397 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:57:25,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:27,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:31,027 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:57:31,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:32,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:36,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:37,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:41,105 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:57:41,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:41,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:43,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:46,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:48,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:51,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:57:52,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:53,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:57:53,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:57:57,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:57:58,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:02,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:03,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:58:03,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:07,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:08,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:13,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:13,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:13,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:58:14,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:18,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:19,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:23,398 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:58:23,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:25,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:25,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:58:28,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:30,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:34,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:35,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:37,668 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:58:39,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:40,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:44,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:45,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:47,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:58:49,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:51,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:53,399 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:58:56,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:58:56,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:58:59,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:01,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:01,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:06,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:06,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:09,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:11,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:11,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:12,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:17,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:18,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:22,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:22,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:23,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:23,399 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:59:28,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:28,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:33,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:33,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:33,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:38,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:38,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:43,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 21:59:43,349 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 21:59:43,349 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 21:59:43,351 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 21:59:44,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:44,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:44,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 21:59:44,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:45,236 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:47,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:49,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:49,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 21:59:49,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:53,400 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 21:59:54,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 21:59:55,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 21:59:59,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:00,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:04,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:05,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:09,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:10,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:14,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:15,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:19,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:20,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:23,401 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:00:24,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:26,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:29,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:31,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:34,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:36,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:39,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:41,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:44,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:46,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:49,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:51,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:53,401 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:00:54,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:00:57,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:00:59,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:02,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:04,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:07,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:09,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:12,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:14,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:17,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:19,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:22,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:23,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:01:24,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:28,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:29,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:33,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:34,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:38,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:39,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:43,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:44,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:48,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:49,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:53,402 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:01:54,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:54,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:01:59,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:01:59,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:04,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:04,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:09,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:09,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:14,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:14,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:19,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:19,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:23,403 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:02:24,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:25,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:29,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:30,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:34,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:35,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:39,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:40,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:44,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:45,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:49,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:50,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:53,403 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:02:54,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:02:56,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:02:59,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:01,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:04,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:06,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:09,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:11,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:14,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:16,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:19,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:21,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:23,405 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:03:24,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:27,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:29,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:32,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:34,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:37,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:39,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:44,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:47,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:49,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:52,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:53,404 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:03:54,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:03:58,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:03:59,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:03,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:08,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:09,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:13,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:14,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:18,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:19,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:23,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:04:24,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:24,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:29,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:29,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:34,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:34,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:39,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:39,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:44,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:44,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:49,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:49,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:53,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:04:54,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:04:55,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:04:59,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:00,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:04,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:05,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:09,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:10,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:14,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:15,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:19,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:20,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:23,406 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:05:24,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:26,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:29,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:31,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:34,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:36,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:39,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:41,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:44,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:46,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:49,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:51,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:53,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:05:54,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:05:57,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:05:59,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:02,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:04,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:07,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:09,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:12,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:14,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:17,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:19,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:22,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:23,407 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:06:24,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:28,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:29,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:33,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:34,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:38,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:39,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:43,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:44,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:48,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:49,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:53,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:06:54,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:54,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:06:59,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:06:59,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:04,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:04,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:09,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:09,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:14,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:14,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:19,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:19,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:23,408 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:07:24,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:25,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:29,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:30,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:34,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:35,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:39,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:40,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:44,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:45,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:49,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:50,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:53,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:07:54,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:07:56,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:07:59,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:01,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:04,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:06,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:09,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:11,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:14,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:16,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:19,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:21,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:23,409 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:08:24,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:27,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:29,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:32,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:34,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:37,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:39,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:42,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:44,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:47,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:49,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:52,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:53,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:08:54,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:08:58,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:08:59,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:03,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:04,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:08,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:09,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:13,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:14,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:18,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:19,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:23,410 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:09:24,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:24,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:29,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:29,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:34,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:34,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:39,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:39,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:44,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:44,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:49,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:49,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:53,411 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:09:54,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:09:55,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:09:59,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:00,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:04,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:05,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:09,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:10,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:14,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:15,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:19,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:20,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:23,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:10:24,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:26,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:29,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:31,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:34,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:36,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:39,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:41,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:44,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:46,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:49,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:51,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:53,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:10:54,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:10:57,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:10:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:02,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:04,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:07,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:09,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:12,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:14,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:17,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:19,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:22,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:23,412 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:11:24,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:28,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:29,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:33,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:34,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:38,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:39,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:43,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:44,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:48,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:49,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:53,413 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:11:54,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:11:54,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:11:58,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:11:58,070 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:11:58,071 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:11:58,074 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:11:58,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:11:59,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:11:59,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:00,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:00,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:12:04,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:05,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:09,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:10,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:14,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:15,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:19,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:20,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:23,414 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:12:24,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:25,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:29,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:30,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:32,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:12:34,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:36,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:40,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:40,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:41,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:46,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:46,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:51,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:52,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:53,415 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:12:56,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:12:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:12:58,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:01,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:05,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:06,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:11,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:13,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:13,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:16,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:18,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:21,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:23,416 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:13:24,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:27,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:30,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:32,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:37,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:42,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:42,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:42,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:43,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:13:44,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:13:47,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:52,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:53,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:13:57,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:58,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:58,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:13:58,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:13:59,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:14:03,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:08,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:13,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:18,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:23,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:23,417 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:14:28,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:32,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:14:33,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:38,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:43,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:48,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:53,418 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:14:54,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:14:59,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:04,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:09,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:10,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:15:14,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:19,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:23,428 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:15:24,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:29,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:34,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:39,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:44,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:49,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:53,419 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:15:54,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:15:54,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:15:55,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:15:59,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:00,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:04,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:05,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:09,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:10,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:14,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:15,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:19,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:20,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:23,420 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:16:25,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:25,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:30,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:30,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:35,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:35,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:40,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:40,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:45,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:45,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:50,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:16:50,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:53,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:16:55,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:16:56,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:00,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:01,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:05,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:06,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:08,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:17:10,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:11,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:15,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:16,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:20,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:21,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:23,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:17:27,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:32,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:36,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:36,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:36,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:37,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:39,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:17:42,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:43,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:48,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:48,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:51,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:17:53,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:17:53,421 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:17:53,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:58,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:17:59,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:03,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:18:04,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:04,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:09,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:09,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:14,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:15,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:19,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:18:19,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:20,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:23,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:18:25,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:26,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:26,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:29,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:18:31,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:31,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:36,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:38,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:38,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:41,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:18:41,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:43,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:46,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:49,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:51,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:53,422 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:18:53,691 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:18:54,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:18:57,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:18:59,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:02,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:04,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:07,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:09,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:12,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:14,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:19:14,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:18,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:19,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:23,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:23,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:19:25,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:26,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:19:28,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:30,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:33,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:35,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:38,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:40,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:43,125 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:19:44,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:45,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:49,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:52,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:53,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:19:54,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:19:55,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:19:57,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:19:59,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:03,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:04,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:07,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:20:08,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:09,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:14,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:14,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:14,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:17,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:20:19,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:19,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:23,425 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:20:25,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:25,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:25,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:25,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:29,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:20:31,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:31,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:36,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:38,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:41,597 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:20:42,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:43,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:47,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:50,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:52,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:20:53,426 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:20:53,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:20:55,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:20:57,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:01,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:02,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:05,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:21:06,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:08,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:13,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:13,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:13,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:17,950 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:21:18,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:18,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:23,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:23,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:21:24,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:28,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:29,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:21:29,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:33,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:34,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:39,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:39,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:40,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:21:40,961 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:21:40,963 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:21:40,971 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:21:41,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:21:42,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:21:43,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:21:44,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:44,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:49,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:49,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:53,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:21:54,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:21:55,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:21:57,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:21:59,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:00,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:05,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:05,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:07,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:22:10,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:11,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:16,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:16,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:19,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:22:21,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:21,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:23,431 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:22:27,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:27,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:27,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:31,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:22:32,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:33,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:37,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:38,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:42,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:42,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:43,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:43,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:22:48,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:48,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:53,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:53,433 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:22:54,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:22:57,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:22:58,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:22:59,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:03,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:04,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:08,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:09,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:09,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:23:13,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:14,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:19,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:20,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:23:20,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:23,433 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:23:24,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:26,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:29,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:31,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:23:32,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:34,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:37,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:40,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:43,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:43,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:23:45,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:48,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:50,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:53,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:23:53,436 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:23:55,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:23:56,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:23:58,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:01,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:03,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:06,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:07,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:24:09,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:11,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:14,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:17,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:24:17,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:17,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:20,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:22,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:23,435 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:24:25,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:27,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:29,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:24:31,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:32,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:36,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:38,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:41,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:24:42,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:43,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:47,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:52,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:53,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:24:53,435 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:24:56,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:24:57,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:24:58,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:02,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:03,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:07,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:08,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:25:08,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:12,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:13,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:17,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:19,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:22,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:25:23,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:23,436 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:25:24,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:28,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:30,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:30,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:33,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:25:34,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:35,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:39,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:40,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:44,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:45,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:49,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:51,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:53,437 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:25:54,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:25:56,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:25:59,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:02,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:02,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:04,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:05,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:26:05,737 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:26:05,738 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:26:05,741 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:26:06,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:26:07,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:26:07,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:08,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:26:09,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:12,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:14,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:18,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:19,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:20,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:26:23,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:23,438 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:26:25,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:29,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:29,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:29,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:30,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:34,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:26:34,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:35,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:39,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:40,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:45,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:45,375 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:26:46,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:50,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:51,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:53,441 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:26:55,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:26:57,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:26:57,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:27:00,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:02,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:06,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:07,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:27:08,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:11,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:13,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:17,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:17,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:18,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:19,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:27:22,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:23,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:23,446 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:27:27,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:28,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:31,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:27:32,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:33,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:37,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:38,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:43,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:43,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:27:44,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:48,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:49,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:53,447 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:27:53,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:27:53,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:27:55,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:27:58,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:00,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:03,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:06,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:28:06,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:08,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:11,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:14,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:16,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:18,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:28:19,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:21,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:23,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:28:25,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:27,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:30,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:30,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:28:32,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:35,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:37,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:40,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:42,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:45,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:46,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:28:48,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:50,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:53,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:53,450 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:28:55,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:28:58,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:28:58,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:29:00,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:03,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:06,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:07,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:29:08,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:11,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:13,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:16,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:16,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:17,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:29:19,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:21,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:23,451 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:29:24,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:27,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:29,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:29:29,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:32,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:34,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:37,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:39,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:42,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:43,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:29:45,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:47,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:50,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:53,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:29:53,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:29:55,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:29:55,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:29:58,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:00,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:03,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:03,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:04,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:30:04,024 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:30:04,025 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:30:04,027 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:30:04,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:30:05,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:30:06,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:06,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:30:10,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:11,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:15,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:16,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:18,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:30:20,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:21,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:23,452 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:30:26,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:26,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:30,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:30:31,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:31,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:36,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:36,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:40,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:30:41,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:42,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:46,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:47,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:52,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:30:52,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:52,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:30:53,453 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:30:57,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:30:58,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:02,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:02,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:03,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:04,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:31:08,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:08,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:13,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:13,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:14,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:31:18,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:18,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:23,454 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:31:23,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:31:24,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:24,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:29,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:29,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:34,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:34,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:35,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:31:39,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:40,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:44,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:45,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:47,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:31:50,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:50,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:53,455 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:31:55,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:31:56,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:31:58,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:32:00,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:01,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:05,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:07,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:07,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:10,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:32:11,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:12,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:16,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:19,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:19,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:19,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:21,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:22,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:32:23,457 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:32:24,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:26,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:30,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:31,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:34,467 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:32:35,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:36,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:40,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:41,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:45,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:46,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:32:47,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:50,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:52,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:53,471 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:32:55,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:32:57,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:32:58,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:00,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:02,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:06,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:06,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:07,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:08,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:11,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:12,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:17,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:17,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:19,925 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:22,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:22,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:23,458 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:33:27,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:28,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:32,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:32,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:34,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:37,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:39,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:43,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:44,212 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:45,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:48,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:50,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:53,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:33:53,460 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:33:54,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:33:54,676 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:33:54,677 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:33:54,681 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:33:55,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:33:55,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:33:56,307 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:57,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:33:58,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:00,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:03,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:05,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:08,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:34:09,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:11,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:14,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:16,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:20,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:20,509 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:34:22,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:23,460 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:34:25,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:27,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:30,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:34:30,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:32,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:35,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:37,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:41,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:41,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:42,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:42,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:34:46,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:47,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:51,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:51,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:53,461 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:34:53,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:34:54,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:34:56,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:34:58,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:03,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:03,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:06,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:35:08,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:09,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:13,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:14,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:16,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:35:18,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:19,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:23,463 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:35:23,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:23,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:24,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:27,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:35:28,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:29,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:34,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:34,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:36,086 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:35:39,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:40,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:45,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:45,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:48,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:35:50,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:51,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:53,464 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:35:55,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:55,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:35:56,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:35:58,274 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:36:00,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:02,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:06,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:07,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:08,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:36:11,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:12,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:16,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:17,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:20,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:36:21,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:23,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:23,465 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:36:26,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:28,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:30,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:36:31,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:33,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:36,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:38,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:41,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:42,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:36:44,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:46,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:49,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:52,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:52,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:36:53,466 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:36:54,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:36:57,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:36:59,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:03,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:04,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:04,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:37:08,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:09,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:13,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:14,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:16,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:37:18,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:20,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:23,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:37:24,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:24,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:25,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:29,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:37:29,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:31,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:35,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:36,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:37,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:37:37,308 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:37:37,309 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:37:37,311 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:37:38,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:37:39,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:37:40,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:41,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:46,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:47,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:50,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:37:52,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:37:52,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:53,467 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:37:57,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:37:57,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:02,302 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:38:02,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:03,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:07,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:08,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:12,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:13,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:16,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:38:17,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:18,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:22,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:23,469 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:38:24,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:26,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:38:28,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:29,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:33,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:35,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:38,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:38:40,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:43,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:46,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:48,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:50,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:38:51,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:38:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:53,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:38:58,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:38:58,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:00,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:39:03,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:03,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:08,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:08,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:14,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:14,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:14,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:39:19,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:19,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:23,470 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:39:24,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:24,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:24,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:25,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:39:29,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:29,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:34,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:35,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:37,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:39:39,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:40,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:45,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:45,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:47,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:39:50,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:50,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:53,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:39:55,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:55,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:39:56,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:39:59,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:40:00,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:01,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:05,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:06,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:10,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:40:11,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:12,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:17,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:17,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:20,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:40:22,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:23,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:23,472 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:40:28,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:28,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:30,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:40:33,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:33,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:38,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:38,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:38,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:38,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:40,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:40:43,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:43,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:48,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:49,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:52,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:40:53,473 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:40:54,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:54,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:40:59,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:40:59,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:04,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:04,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:09,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:09,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:14,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:14,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:19,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:19,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:23,476 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:41:24,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:24,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:24,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:25,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:29,155 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:41:29,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:31,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:34,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:36,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:39,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:41:40,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:41,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:45,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:46,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:50,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:41:50,256 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:41:50,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:41:50,263 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:41:50,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:41:50,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:51,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:41:52,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:41:53,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:41:55,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:41:57,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:01,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:02,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:03,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:42:06,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:07,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:12,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:13,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:13,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:42:17,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:18,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:22,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:23,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:23,479 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:42:24,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:42:27,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:28,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:33,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:33,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:36,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:42:38,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:39,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:43,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:44,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:48,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:42:49,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:49,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:53,477 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:42:54,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:55,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:42:58,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:42:59,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:42:59,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:01,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:04,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:06,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:10,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:10,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:11,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:43:12,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:15,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:17,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:20,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:21,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:43:22,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:23,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:43:25,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:28,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:31,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:31,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:43:34,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:36,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:39,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:41,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:41,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:41,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:44,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:45,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:43:46,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:49,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:52,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:43:53,478 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:43:55,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:43:55,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:43:57,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:00,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:02,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:05,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:44:05,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:07,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:10,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:12,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:12,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:12,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:15,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:44:16,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:17,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:21,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:22,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:23,480 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:44:25,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:44:26,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:27,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:31,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:32,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:36,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:37,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:44:38,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:41,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:43,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:46,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:46,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:44:48,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:51,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:53,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:53,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:44:57,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:44:58,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:44:58,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:02,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:03,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:07,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:09,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:09,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:12,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:14,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:17,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:19,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:19,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:22,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:23,481 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:45:24,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:28,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:29,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:33,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:34,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:36,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:45:36,182 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:45:36,183 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:45:36,185 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:45:36,210 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:45:37,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:38,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:39,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:39,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:44,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:44,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:49,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:49,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:50,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:53,482 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:45:54,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:45:55,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:45:59,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:45:59,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:00,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:04,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:05,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:10,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:11,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:11,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:46:15,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:16,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:20,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:21,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:21,551 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:46:23,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:46:26,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:26,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:31,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:31,630 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:46:32,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:32,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:36,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:37,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:42,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:43,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:43,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:46:47,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:48,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:52,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:53,483 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:46:53,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:46:53,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:46:58,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:46:58,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:03,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:03,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:47:04,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:08,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:09,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:14,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:14,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:15,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:47:19,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:19,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:23,484 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:47:24,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:25,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:47:25,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:29,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:30,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:35,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:35,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:47:35,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:40,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:40,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:45,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:45,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:47:46,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:50,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:51,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:47:53,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:47:55,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:47:55,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:47:56,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:00,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:01,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:05,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:06,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:07,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:48:10,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:11,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:15,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:48:16,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:16,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:21,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:21,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:48:22,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:23,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:48:26,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:28,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:29,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:48:31,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:33,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:37,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:38,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:39,634 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:48:42,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:43,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:47,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:47,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:48,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:51,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:48:53,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:48:53,487 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:48:53,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:48:58,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:01,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:03,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:04,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:49:08,950 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:49:08,951 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:49:08,953 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:49:09,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:09,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:49:09,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:09,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:11,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:14,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:14,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:20,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:20,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:21,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:23,490 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:49:25,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:26,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:31,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:32,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:32,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:36,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:37,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:41,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:42,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:43,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:46,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:48,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:51,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:49:52,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:49:53,489 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:49:53,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:49:58,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:01,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:50:03,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:04,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:08,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:09,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:11,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:50:13,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:14,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:19,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:20,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:23,489 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:50:23,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:50:24,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:25,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:29,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:33,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:50:34,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:36,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:39,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:42,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:45,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:45,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:50:47,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:50,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:52,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:50:53,491 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:50:55,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:50:55,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:50:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:00,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:03,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:03,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:06,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:07,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:51:08,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:11,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:13,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:16,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:18,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:20,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:51:22,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:23,491 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:51:23,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:27,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:28,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:32,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:51:33,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:33,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:38,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:38,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:42,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:51:43,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:43,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:48,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:50,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:53,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:51:53,492 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:51:53,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:51:55,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:51:58,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:01,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:01,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:04,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:05,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:52:06,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:09,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:12,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:14,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:15,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:52:17,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:19,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:22,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:23,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:52:24,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:28,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:29,680 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:52:30,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:33,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:35,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:38,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:39,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:52:40,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:43,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:45,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:49,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:49,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:52:49,541 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:52:49,542 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:52:49,546 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:52:49,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:52:50,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:50,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:52:51,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:52:53,494 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:52:54,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:52:56,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:52:59,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:02,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:53:02,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:04,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:07,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:09,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:12,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:14,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:53:14,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:17,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:19,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:22,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:23,495 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:53:25,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:26,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:53:27,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:30,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:32,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:36,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:37,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:38,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:53:41,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:42,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:46,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:47,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:49,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:53:51,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:52,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:53,496 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:53:57,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:53:58,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:53:59,621 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:54:02,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:03,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:08,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:09,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:09,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:54:13,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:14,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:19,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:19,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:19,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:21,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:54:23,497 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:54:24,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:24,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:29,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:29,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:31,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:54:34,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:35,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:40,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:41,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:44,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:54:45,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:46,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:50,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:52,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:54:53,498 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:54:56,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:54:56,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:54:57,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:01,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:03,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:03,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:06,221 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:55:06,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:08,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:11,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:14,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:16,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:18,312 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:55:19,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:21,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:23,500 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:55:25,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:26,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:28,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:55:30,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:31,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:35,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:36,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:40,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:40,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:55:42,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:45,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:47,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:50,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:50,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:55:52,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:55:53,501 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:55:55,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:55:58,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:01,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:01,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:03,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:06,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:08,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:11,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:11,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:14,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:16,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:19,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:21,821 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:22,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:23,501 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:56:24,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:27,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:29,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:32,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 22:56:32,377 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 22:56:32,378 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 22:56:32,381 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 22:56:32,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:33,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 22:56:34,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:35,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:35,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:38,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:40,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:43,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:45,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:46,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:48,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:50,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:53,502 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:56:54,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:56:55,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:56:58,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:56:59,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:01,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:05,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:06,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:10,361 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:57:10,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:11,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:16,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:17,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:17,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:17,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:20,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:57:21,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:22,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:23,503 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:57:27,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:28,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:32,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:57:32,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:33,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:37,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:39,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:42,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:57:43,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:44,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:48,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:49,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:53,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:57:54,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:54,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:57:54,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:57:59,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:57:59,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:03,814 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:58:05,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:05,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:05,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:10,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:10,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:15,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:15,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:15,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:58:20,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:20,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:23,505 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:58:25,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:25,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:25,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:58:30,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:30,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:35,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:36,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:58:36,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:36,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:40,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:41,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:45,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:47,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:50,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:58:51,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:53,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:58:53,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:53,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:58:57,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:58:58,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:02,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:59:03,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:03,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:08,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:09,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:12,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:59:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:14,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:18,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:19,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:23,506 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:59:24,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:59:24,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:25,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:29,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:32,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:34,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:36,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:59:37,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:39,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:43,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:44,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:46,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:59:48,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:49,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:53,508 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 22:59:54,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 22:59:55,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 22:59:58,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 22:59:59,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:00,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:04,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:05,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:09,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:10,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:00:10,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:14,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:15,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:19,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:00:19,296 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:00:19,296 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:00:19,299 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:00:19,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:20,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:00:20,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:00:21,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:00:21,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:23,508 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:00:24,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:26,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:30,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:31,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:32,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:00:35,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:36,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:40,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:41,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:44,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:00:45,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:46,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:50,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:51,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:00:53,511 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:00:55,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:00:56,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:00:56,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:00,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:01,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:05,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:05,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:06,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:08,549 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:01:10,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:11,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:15,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:17,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:18,644 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:01:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:22,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:23,511 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:01:26,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:27,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:30,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:01:31,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:32,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:37,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:37,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:37,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:40,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:01:42,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:43,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:47,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:48,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:53,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:53,512 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:01:53,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:01:54,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:01:58,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:01:58,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:03,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:03,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:08,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:09,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:10,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:02:13,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:14,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:18,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:19,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:22,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:02:23,512 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:02:24,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:24,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:29,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:29,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:32,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:02:34,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:35,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:40,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:40,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:44,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:02:45,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:45,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:50,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:51,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:02:53,513 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:02:54,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:02:55,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:02:56,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:00,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:01,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:05,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:06,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:08,593 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:03:10,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:11,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:15,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:16,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:18,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:03:21,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:21,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:23,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:03:26,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:27,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:30,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:03:32,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:32,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:37,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:38,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:38,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:43,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:03:43,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:47,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:48,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:52,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:53,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:03:54,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:03:55,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:03:57,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:03:59,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:02,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:05,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:07,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:08,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:04:10,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:12,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:16,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:17,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:18,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:04:18,334 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:04:18,335 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:04:18,338 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:04:18,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:04:19,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:04:20,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:04:21,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:23,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:23,514 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:04:28,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:28,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:28,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:32,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:04:33,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:34,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:39,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:40,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:40,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:40,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:42,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:04:44,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:45,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:51,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:04:53,515 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:04:53,666 WARNING FileStreamThread:607337 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream. func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream',), kwargs: {'json': {'files': {'wandb-events.jsonl': {'offset': 4326, 'content': ['{"system.gpu.0.memoryAllocated": 79.71, "system.gpu.process.0.memoryAllocated": 79.71, "system.gpu.0.memoryAllocatedBytes": 13687805815.47, "system.gpu.process.0.memoryAllocatedBytes": 13687805815.47, "system.gpu.0.memory": 20.0, "system.gpu.process.0.memory": 20.0, "system.gpu.0.gpu": 48.67, "system.gpu.process.0.gpu": 48.67, "system.gpu.0.temp": 44.47, "system.gpu.process.0.temp": 44.47, "system.gpu.0.powerWatts": 121.37, "system.gpu.process.0.powerWatts": 121.37, "system.gpu.0.powerPercent": 35.73, "system.gpu.process.0.powerPercent": 35.73, "system.network.sent": 40079727480.87, "system.network.recv": 108455167699.8, "system.proc.memory.availableMB": 3500.83, "system.memory": 93.76, "system.proc.memory.rssMB": 48327.5, "system.proc.memory.percent": 85.92, "system.cpu": 27.34, "system.cpu.0.cpu_percent": 25.13, "system.cpu.1.cpu_percent": 51.91, "system.cpu.2.cpu_percent": 33.92, "system.cpu.3.cpu_percent": 31.69, "system.cpu.4.cpu_percent": 28.47, "system.cpu.5.cpu_percent": 25.37, "system.cpu.6.cpu_percent": 33.07, "system.cpu.7.cpu_percent": 25.93, "system.proc.cpu.threads": 60, "system.disk.\\\\.usagePercent": 3.9, "system.disk.\\\\.usageGB": 37.45, "system.disk.in": 2047.67, "system.disk.out": 8520.88, "_wandb": true, "_timestamp": 1700514203.513119, "_runtime": 129854.479032}', '{"system.gpu.0.memoryAllocated": 79.54, "system.gpu.process.0.memoryAllocated": 79.54, "system.gpu.0.memoryAllocatedBytes": 13658972160.0, "system.gpu.process.0.memoryAllocatedBytes": 13658972160.0, "system.gpu.0.memory": 23.33, "system.gpu.process.0.memory": 23.33, "system.gpu.0.gpu": 53.2, "system.gpu.process.0.gpu": 53.2, "system.gpu.0.temp": 44.8, "system.gpu.process.0.temp": 44.8, "system.gpu.0.powerWatts": 104.02, "system.gpu.process.0.powerWatts": 104.02, "system.gpu.0.powerPercent": 30.57, "system.gpu.process.0.powerPercent": 30.57, "system.network.sent": 40080350904.2, "system.network.recv": 108459703433.87, "system.proc.memory.availableMB": 3462.37, "system.memory": 93.83, "system.proc.memory.rssMB": 48362.06, "system.proc.memory.percent": 85.99, "system.cpu": 29.14, "system.cpu.0.cpu_percent": 26.6, "system.cpu.1.cpu_percent": 53.33, "system.cpu.2.cpu_percent": 32.67, "system.cpu.3.cpu_percent": 26.36, "system.cpu.4.cpu_percent": 31.14, "system.cpu.5.cpu_percent": 35.77, "system.cpu.6.cpu_percent": 47.15, "system.cpu.7.cpu_percent": 19.83, "system.proc.cpu.threads": 60, "system.disk.\\\\.usagePercent": 3.9, "system.disk.\\\\.usageGB": 37.45, "system.disk.in": 2047.67, "system.disk.out": 8521.32, "_wandb": true, "_timestamp": 1700514233.513694, "_runtime": 129884.479607}']}, 'output.log': {'offset': 827, 'content': ['ERROR 2023-11-20T21:03:42.213619 30%|███████████████████████████▋ | 18237/60000 [36:04:25<134:53:03, 11.63s/it]\n']}}, 'dropped': 0}} -2023-11-20 23:04:54,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:04:55,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:04:56,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:00,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:01,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:04,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:05:05,620 WARNING FileStreamThread:607337 [file_stream.py:request_with_retry():668] requests_with_retry encountered retryable exception: 502 Server Error: Bad Gateway for url: https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream. func: functools.partial(>, timeout=180), args: ('https://api.wandb.ai/files/finnish-nlp/huggingface/nwx0xdmm/file_stream',), kwargs: {'json': {'files': {'output.log': {'offset': 827, 'content': ['ERROR 2023-11-20T21:04:54.301230 30%|███████████████████████████▋ | 18243/60000 [36:05:38<134:33:27, 11.60s/it]\n']}}, 'dropped': 0}} -2023-11-20 23:05:05,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:06,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:10,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:11,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:16,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:16,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:16,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:05:21,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:21,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:23,519 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:05:26,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:27,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:29,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:05:31,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:32,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:36,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:37,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:41,168 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:05:42,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:42,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:47,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:48,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:51,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:05:52,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:53,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:05:53,517 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:05:58,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:05:59,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:02,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:06:03,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:04,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:08,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:09,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:13,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:14,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:06:15,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:15,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:18,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:20,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:23,521 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:06:24,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:26,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:26,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:06:29,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:31,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:34,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:36,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:06:36,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:40,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:41,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:45,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:46,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:06:47,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:51,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:52,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:53,519 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:06:56,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:06:58,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:06:58,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:07:01,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:03,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:06,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:08,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:11,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:07:12,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:13,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:17,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:19,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:22,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:23,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:07:23,520 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:07:24,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:27,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:29,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:29,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:33,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:33,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:07:34,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:38,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:40,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:43,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:07:43,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:45,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:48,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:51,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:53,523 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:07:54,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:07:54,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:07:56,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:07:59,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:01,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:03,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:08:03,282 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:08:03,283 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:08:03,287 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:08:03,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:08:04,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:08:05,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:05,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:08:06,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:10,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:11,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:15,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:16,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:08:17,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:20,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:22,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:23,524 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:08:26,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:28,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:28,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:08:31,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:33,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:36,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:38,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:08:39,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:42,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:44,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:47,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:49,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:51,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:08:52,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:08:53,524 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:08:54,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:08:58,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:00,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:01,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:09:03,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:05,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:08,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:11,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:13,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:15,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:09:16,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:18,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:22,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:23,526 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:09:23,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:25,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:09:27,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:28,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:33,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:33,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:37,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:09:38,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:39,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:44,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:44,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:44,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:44,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:47,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:09:49,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:50,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:53,528 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:09:55,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:09:55,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:09:58,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:10:00,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:01,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:05,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:06,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:11,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:11,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:12,782 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:10:16,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:16,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:22,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:22,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:22,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:10:23,527 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:10:27,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:27,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:32,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:32,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:32,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:10:33,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:37,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:38,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:42,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:43,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:45,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:10:47,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:48,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:53,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:10:53,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:53,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:10:57,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:10:58,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:10:58,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:03,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:03,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:03,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:07,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:11:08,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:09,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:14,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:14,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:17,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:11:19,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:20,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:23,530 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:11:25,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:25,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:27,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:11:30,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:31,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:35,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:36,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:39,552 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:11:40,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:41,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:45,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:46,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:49,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:11:49,136 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:11:49,137 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:11:49,140 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:11:49,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:11:50,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:51,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:11:52,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:11:53,531 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:11:55,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:11:57,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:01,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:02,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:03,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:12:06,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:07,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:12,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:12,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:15,021 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:12:17,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:18,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:22,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:23,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:23,533 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:12:27,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:12:27,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:28,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:32,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:33,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:38,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:38,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:39,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:12:43,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:43,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:48,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:48,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:48,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:49,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:12:53,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:53,533 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:12:54,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:12:58,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:12:59,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:01,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:13:03,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:04,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:08,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:09,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:13,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:14,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:18,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:19,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:23,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:13:23,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:25,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:27,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:13:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:30,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:35,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:39,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:39,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:13:40,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:44,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:45,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:49,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:49,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:50,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:51,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:13:53,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:13:54,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:13:56,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:13:59,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:01,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:03,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:14:04,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:06,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:10,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:11,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:13,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:14:15,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:16,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:21,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:21,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:21,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:23,535 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:14:25,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:14:26,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:27,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:32,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:32,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:35,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:14:38,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:38,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:42,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:43,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:48,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:48,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:49,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:49,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:14:53,536 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:14:53,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:54,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:14:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:14:59,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:15:00,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:05,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:08,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:10,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:11,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:15:13,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:15,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:19,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:19,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:19,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:20,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:21,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:15:23,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:15:24,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:25,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:30,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:30,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:33,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:15:35,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:35,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:40,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:40,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:45,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:45,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:15:46,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:50,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:51,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:53,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:15:55,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:15:55,432 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:15:55,432 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:15:55,436 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:15:55,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:15:56,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:15:56,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:15:57,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:16:01,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:01,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:06,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:06,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:08,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:16:11,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:12,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:17,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:17,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:18,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:16:22,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:22,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:23,539 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:16:27,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:27,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:29,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:16:32,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:33,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:37,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:38,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:41,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:16:42,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:44,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:48,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:49,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:51,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:16:53,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:53,540 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:16:54,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:16:58,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:16:59,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:01,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:17:03,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:05,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:09,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:10,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:11,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:17:14,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:15,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:19,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:19,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:19,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:20,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:23,541 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:17:23,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:17:24,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:26,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:29,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:31,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:35,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:35,686 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:17:37,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:40,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:42,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:45,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:45,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:17:47,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:50,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:52,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:17:53,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:17:55,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:17:57,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:17:58,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:03,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:05,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:07,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:18:08,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:11,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:13,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:17,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:18,005 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:18:19,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:22,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:23,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:18:25,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:27,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:30,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:18:30,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:32,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:35,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:37,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:40,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:18:40,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:42,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:45,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:47,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:51,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:52,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:18:53,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:18:53,543 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:18:56,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:18:58,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:01,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:19:01,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:03,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:06,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:08,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:11,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:14,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:16,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:17,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:19:19,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:21,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:23,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:19:25,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:26,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:27,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:19:30,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:31,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:35,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:36,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:39,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:19:39,334 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:19:39,334 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:19:39,337 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:19:39,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:19:40,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:19:41,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:41,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:19:42,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:46,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:47,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:51,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:51,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:19:52,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:19:53,546 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:19:56,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:19:58,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:01,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:20:02,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:04,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:07,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:09,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:12,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:20:12,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:14,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:17,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:19,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:22,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:20:22,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:23,547 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:20:25,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:27,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:30,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:32,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:34,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:20:36,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:37,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:41,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:43,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:46,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:20:46,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:48,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:51,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:53,550 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:20:54,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:54,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:20:57,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:20:58,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:20:59,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:02,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:05,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:08,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:21:08,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:10,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:13,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:15,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:18,530 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:21:19,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:20,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:23,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:21:24,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:25,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:29,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:21:30,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:31,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:35,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:36,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:40,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:41,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:43,749 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:21:45,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:46,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:50,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:51,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:21:53,549 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:21:53,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:21:56,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:21:56,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:01,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:02,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:05,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:22:07,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:07,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:12,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:13,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:16,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:22:17,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:18,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:22,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:23,551 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:22:24,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:24,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:28,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:22:28,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:29,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:33,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:35,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:38,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:22:38,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:40,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:43,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:45,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:48,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:50,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:22:50,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:53,552 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:22:54,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:22:55,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:22:59,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:01,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:02,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:23:05,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:06,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:10,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:11,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:14,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:23:15,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:16,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:20,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:21,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:21,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:21,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:23,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:23:23,245 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:23:23,246 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:23:23,248 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:23:23,563 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:23:23,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:23:24,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:23:26,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:26,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:31,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:32,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:35,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:23:36,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:37,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:41,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:43,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:45,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:23:47,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:48,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:52,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:53,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:23:54,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:54,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:23:57,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:23:57,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:23:59,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:02,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:05,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:07,981 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:24:08,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:10,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:13,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:15,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:18,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:20,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:24:20,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:23,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:24:24,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:25,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:30,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:24:30,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:31,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:35,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:36,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:40,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:41,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:42,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:24:45,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:46,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:50,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:52,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:24:53,554 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:24:54,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:24:56,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:24:57,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:01,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:02,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:06,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:07,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:11,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:12,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:16,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:17,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:21,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:23,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:23,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:23,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:23,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:25:26,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:25:27,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:28,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:32,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:33,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:37,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:37,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:25:39,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:42,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:44,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:47,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:49,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:25:49,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:53,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:25:53,557 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:25:54,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:25:58,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:00,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:26:00,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:04,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:05,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:09,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:10,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:12,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:26:15,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:15,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:20,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:21,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:23,558 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:26:24,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:26:25,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:26,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:30,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:31,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:34,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:26:36,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:36,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:41,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:41,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:41,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:44,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:26:46,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:46,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:51,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:26:52,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:53,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:26:54,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:26:57,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:26:57,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:02,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:02,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:06,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:27:07,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:08,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:13,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:13,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:13,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:16,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:27:18,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:18,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:23,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:23,559 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:27:24,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:29,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:29,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:27:29,439 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:27:29,440 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:27:29,442 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:27:29,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:27:30,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:30,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:27:34,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:35,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:39,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:39,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:41,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:42,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:27:44,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:46,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:50,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:52,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:27:52,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:27:53,560 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:27:55,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:27:57,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:01,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:02,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:28:02,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:06,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:07,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:12,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:12,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:12,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:14,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:28:17,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:17,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:22,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:23,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:23,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:28:26,243 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:28:28,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:28,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:33,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:35,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:38,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:28:38,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:40,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:43,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:46,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:48,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:28:48,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:51,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:53,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:28:53,562 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:28:57,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:28:58,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:00,496 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:02,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:03,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:07,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:08,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:09,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:12,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:14,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:14,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:17,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:19,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:23,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:23,563 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:29:25,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:26,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:28,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:30,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:33,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:36,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:36,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:38,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:41,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:44,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:46,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:46,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:49,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:51,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:53,564 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:29:54,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:29:56,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:29:58,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:29:59,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:01,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:04,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:06,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:09,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:11,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:30:12,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:14,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:17,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:20,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:21,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:30:22,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:23,565 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:30:25,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:28,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:30,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:31,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:30:34,182 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:35,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:39,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:40,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:30:41,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:44,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:46,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:49,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:51,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:30:52,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:30:53,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:30:55,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:30:56,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:00,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:02,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:02,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:31:02,489 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:31:02,490 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:31:02,494 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:31:03,421 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:31:04,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:05,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:06,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:07,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:11,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:12,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:12,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:14,512 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:16,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:17,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:21,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:22,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:23,566 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:31:24,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:27,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:27,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:32,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:32,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:34,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:37,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:38,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:43,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:43,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:43,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:46,744 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:48,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:49,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:53,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:31:54,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:54,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:54,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:31:54,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:31:56,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:31:59,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:00,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:04,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:05,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:06,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:32:09,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:10,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:14,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:16,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:32:19,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:21,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:23,568 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:32:24,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:24,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:27,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:29,069 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:32:29,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:32,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:34,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:37,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:40,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:41,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:32:42,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:45,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:47,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:51,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:32:53,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:32:53,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:32:56,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:32:58,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:01,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:03,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:33:03,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:06,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:08,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:11,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:11,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:13,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:33:14,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:16,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:19,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:21,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:23,569 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:33:24,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:25,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:33:26,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:29,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:32,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:34,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:33:34,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:37,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:39,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:42,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:42,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:44,662 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:33:45,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:47,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:50,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:52,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:33:53,571 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:33:55,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:33:56,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:33:57,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:01,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:02,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:06,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:06,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:34:07,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:11,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:12,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:16,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:18,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:18,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:34:21,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:23,572 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:34:23,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:27,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:29,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:34:29,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:32,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:34,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:37,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:38,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:34:38,209 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:34:38,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:34:38,212 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:34:39,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:34:39,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:34:40,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:43,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:45,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:48,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:50,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:51,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:34:53,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:34:53,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:34:55,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:34:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:01,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:35:04,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:06,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:09,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:11,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:13,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:35:15,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:16,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:20,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:22,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:23,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:35:23,573 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:35:25,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:27,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:30,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:32,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:35,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:38,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:41,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:41,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:35:43,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:46,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:48,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:51,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:53,575 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:35:54,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:35:54,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:35:56,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:35:59,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:01,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:04,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:06,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:36:07,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:09,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:12,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:14,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:14,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:16,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:36:17,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:19,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:22,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:23,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:36:25,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:26,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:36:27,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:30,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:32,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:36,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:37,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:39,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:36:41,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:42,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:46,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:47,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:51,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:52,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:36:53,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:36:53,576 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:36:56,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:36:58,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:02,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:03,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:37:04,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:07,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:09,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:12,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:13,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:37:15,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:17,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:20,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:23,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:23,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:37:23,577 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:37:25,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:28,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:30,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:33,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:35,536 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:37:36,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:38,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:41,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:43,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:43,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:45,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:37:46,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:48,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:51,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:53,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:37:55,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:37:56,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:37:59,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:00,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:01,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:06,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:06,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:09,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:11,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:12,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:16,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:17,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:19,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:21,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:22,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:23,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:38:26,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:27,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:27,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:38:27,964 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:38:27,964 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:38:27,967 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:38:28,934 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:38:28,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:29,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:31,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:32,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:37,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:37,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:39,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:42,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:43,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:47,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:48,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:51,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:38:53,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:53,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:38:53,579 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:38:58,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:38:58,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:03,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:03,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:05,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:39:08,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:08,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:13,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:14,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:15,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:39:19,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:19,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:23,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:39:24,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:24,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:25,385 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:39:29,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:29,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:34,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:34,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:37,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:39:39,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:40,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:45,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:45,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:45,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:49,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:39:50,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:50,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:39:53,581 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:39:55,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:39:55,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:00,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:01,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:01,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:01,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:40:05,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:06,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:10,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:12,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:13,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:40:16,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:17,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:21,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:22,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:23,583 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:40:23,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:40:26,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:27,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:31,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:33,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:33,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:35,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:40:37,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:38,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:42,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:43,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:47,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:40:48,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:48,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:53,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:40:53,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:53,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:40:57,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:40:58,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:40:58,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:03,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:04,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:08,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:09,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:11,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:41:14,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:14,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:19,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:19,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:23,584 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:41:23,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:41:24,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:24,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:29,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:31,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:31,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:33,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:41:34,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:36,557 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:39,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:42,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:45,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:45,795 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:41:47,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:50,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:53,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:41:54,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:41:55,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:41:57,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:41:59,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:00,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:04,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:09,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:10,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:14,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:42:14,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:15,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:19,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:20,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:23,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:42:23,301 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:42:23,301 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:42:23,304 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:42:23,586 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:42:24,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:42:24,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:26,143 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:42:26,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:29,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:31,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:35,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:36,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:42:36,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:40,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:41,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:45,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:45,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:46,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:47,344 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:42:50,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:51,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:42:53,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:42:56,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:42:57,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:42:57,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:01,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:02,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:07,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:07,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:11,538 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:43:12,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:13,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:18,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:18,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:18,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:21,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:43:23,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:23,587 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:43:23,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:28,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:28,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:31,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:43:33,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:34,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:39,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:39,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:41,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:43:44,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:44,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:49,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:49,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:43:51,883 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:43:53,593 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:43:54,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:43:55,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:00,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:01,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:01,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:01,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:03,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:44:06,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:06,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:11,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:11,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:16,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:44:16,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:16,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:21,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:21,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:23,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:44:26,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:27,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:28,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:44:31,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:32,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:37,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:37,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:38,267 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:44:42,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:42,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:47,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:48,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:44:48,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:52,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:53,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:53,595 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:44:58,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:44:58,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:44:59,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:45:03,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:03,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:08,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:08,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:11,595 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:45:13,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:13,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:18,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:19,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:21,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:45:23,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:45:24,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:24,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:29,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:29,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:31,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:45:34,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:35,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:40,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:40,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:43,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:45:45,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:45,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:50,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:51,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:45:53,596 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:45:53,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:45:55,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:45:56,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:00,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:02,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:02,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:02,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:03,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:46:03,805 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:46:03,806 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:46:03,808 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:46:03,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:46:05,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:05,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:46:07,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:10,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:12,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:16,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:46:16,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:17,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:21,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:23,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:23,598 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:46:26,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:46:26,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:31,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:33,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:33,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:33,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:36,226 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:46:37,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:38,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:42,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:43,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:46,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:46:47,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:48,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:52,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:53,599 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:46:54,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:46:58,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:46:58,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:46:59,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:03,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:05,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:05,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:08,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:47:09,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:10,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:14,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:16,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:18,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:47:19,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:21,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:23,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:47:25,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:27,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:29,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:47:30,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:32,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:35,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:37,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:39,748 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:47:41,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:46,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:48,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:49,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:47:51,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:53,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:47:53,600 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:47:56,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:47:59,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:01,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:48:01,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:04,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:10,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:11,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:48:12,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:15,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:17,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:21,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:21,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:22,049 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:48:23,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:23,601 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:48:26,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:28,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:31,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:33,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:34,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:48:36,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:38,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:41,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:43,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:44,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:48:46,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:48,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:51,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:53,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:48:54,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:48:54,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:48:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:48:59,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:03,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:05,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:06,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:49:08,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:10,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:13,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:15,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:16,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:49:18,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:20,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:23,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:23,604 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:49:25,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:26,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:49:28,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:30,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:34,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:34,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:49:34,417 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:49:34,418 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:49:34,421 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:49:34,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:49:36,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:36,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:49:39,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:41,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:44,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:45,801 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:49:46,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:49,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:51,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:49:53,605 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:49:55,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:49:55,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:49:57,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:00,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:02,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:05,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:05,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:07,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:50:08,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:11,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:13,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:16,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:18,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:50:18,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:21,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:23,606 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:50:24,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:26,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:28,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:50:29,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:31,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:34,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:36,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:38,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:50:40,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:41,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:45,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:46,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:48,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:50:51,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:51,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:50:53,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:50:56,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:50:57,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:00,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:51:02,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:02,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:07,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:08,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:08,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:12,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:51:12,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:13,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:17,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:19,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:22,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:51:22,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:23,607 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:51:24,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:28,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:30,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:32,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:51:34,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:35,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:39,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:40,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:40,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:42,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:51:45,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:45,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:50,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:50,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:53,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:51:55,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:51:55,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:51:58,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:52:00,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:00,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:05,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:05,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:05,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:08,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:52:10,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:11,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:15,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:16,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:18,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:52:21,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:21,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:23,609 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:52:26,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:26,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:28,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:52:31,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:31,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:36,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:37,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:38,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:52:42,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:42,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:47,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:47,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:50,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:52:52,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:52,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:52:53,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:52:58,124 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:52:58,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:00,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:53:03,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:03,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:08,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:08,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:08,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:08,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:53:08,644 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:53:08,645 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:53:08,648 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:53:09,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:53:10,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:53:13,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:13,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:19,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:19,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:20,373 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:53:23,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:53:24,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:24,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:29,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:30,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:30,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:53:34,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:35,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:40,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:40,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:53:40,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:45,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:46,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:50,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:53:51,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:53:51,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:53,610 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:53:57,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:53:58,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:00,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:54:03,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:03,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:08,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:08,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:08,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:10,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:54:13,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:13,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:19,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:19,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:20,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:54:23,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:54:24,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:24,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:30,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:30,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:30,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:54:35,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:35,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:40,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:40,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:54:41,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:45,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:46,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:50,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:51,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:54:53,611 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:54:55,020 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:54:55,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:54:56,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:01,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:02,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:03,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:55:06,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:07,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:12,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:12,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:13,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:55:13,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:17,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:18,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:22,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:22,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:55:23,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:55:23,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:27,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:28,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:32,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:55:32,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:34,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:37,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:39,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:42,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:55:43,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:44,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:48,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:49,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:53,612 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:55:53,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:55:54,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:55:55,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:55:58,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:00,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:03,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:04,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:56:06,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:08,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:11,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:14,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:14,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:56:16,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:19,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:21,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:23,613 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:56:24,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:24,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:56:27,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:29,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:32,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:33,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-20 23:56:33,215 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-20 23:56:33,216 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-20 23:56:33,220 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-20 23:56:33,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-20 23:56:34,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:56:34,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:38,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:39,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:43,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:45,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:46,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:56:48,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:50,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:53,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:56:53,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:56:55,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:56:59,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:56:59,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:00,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:04,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:06,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:07,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:57:09,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:11,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:14,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:16,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:19,174 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:57:19,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:21,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:23,615 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:57:25,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:27,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:29,247 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:57:31,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:32,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:37,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:37,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:39,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:57:42,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:43,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:48,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:48,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:57:49,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:53,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:57:53,616 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:57:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:55,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:57:58,488 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:57:59,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:00,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:04,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:06,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:08,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:58:09,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:11,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:14,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:16,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:18,627 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:58:19,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:21,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:23,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:58:25,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:27,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:27,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:28,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:58:30,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:32,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:35,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:37,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:38,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:58:41,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:42,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:46,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:58:46,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:48,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:51,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:53,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:58:53,617 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:58:57,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:58:58,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:02,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:59:03,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:03,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:08,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:08,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:13,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:14,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:15,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:59:18,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:19,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:23,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:59:24,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:24,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:25,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:59:29,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:30,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:35,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:59:35,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:35,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:40,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:41,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:41,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:43,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:59:45,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:46,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:50,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:52,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-20 23:59:53,251 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-20 23:59:53,618 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-20 23:59:56,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-20 23:59:57,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:01,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:03,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:00:03,221 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:00:03,222 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:00:03,225 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:00:03,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:00:03,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:05,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:00:07,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:08,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:12,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:14,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:15,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:00:18,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:19,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:23,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:23,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:00:24,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:25,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:00:28,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:31,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:34,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:35,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:00:36,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:39,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:41,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:41,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:43,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:00:45,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:46,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:50,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:52,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:00:53,619 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:00:54,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:00:55,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:00:57,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:00,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:04,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:01:06,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:07,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:11,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:13,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:14,787 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:01:17,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:18,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:22,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:23,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:23,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:01:24,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:01:27,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:28,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:32,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:01:33,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:34,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:38,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:39,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:43,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:44,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:44,992 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:01:48,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:49,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:53,620 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:01:53,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:01:54,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:01:55,064 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:01:58,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:01,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:03,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:02:04,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:06,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:09,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:11,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:11,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:13,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:02:14,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:16,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:19,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:21,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:23,255 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:02:23,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:02:24,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:26,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:29,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:32,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:33,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:02:35,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:37,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:40,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:42,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:43,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:02:45,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:47,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:50,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:52,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:52,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:02:53,470 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:02:53,621 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:02:55,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:02:59,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:01,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:01,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:03:04,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:06,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:09,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:09,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:11,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:03:11,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:14,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:16,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:19,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:03:19,257 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:03:19,257 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:03:19,259 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:03:19,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:03:20,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:21,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:03:22,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:23,622 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:03:25,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:27,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:30,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:32,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:33,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:03:35,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:37,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:41,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:41,811 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:03:43,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:46,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:48,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:51,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:03:52,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:53,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:03:53,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:03:58,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:03:58,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:00,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:04:03,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:04,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:09,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:10,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:10,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:10,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:13,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:04:14,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:15,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:19,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:20,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:23,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:04:23,623 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:04:25,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:25,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:30,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:31,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:35,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:04:35,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:36,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:40,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:42,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:42,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:45,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:04:46,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:47,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:51,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:52,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:04:53,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:04:55,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:04:56,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:04:57,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:01,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:02,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:05,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:05:07,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:12,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:13,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:17,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:05:17,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:18,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:22,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:23,625 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:05:24,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:25,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:05:28,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:29,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:33,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:35,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:38,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:39,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:05:40,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:43,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:45,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:48,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:49,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:05:50,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:53,626 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:05:54,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:05:56,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:56,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:05:57,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:00,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:01,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:05,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:06,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:08,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:10,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:11,611 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:15,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:17,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:17,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:21,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:22,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:23,627 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:06:26,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:27,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:27,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:31,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:33,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:36,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:37,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:38,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:41,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:44,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:44,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:44,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:06:44,154 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:06:44,154 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:06:44,157 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:06:44,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:06:45,263 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:47,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:49,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:52,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:53,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:06:54,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:06:55,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:06:57,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:06:59,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:02,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:04,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:08,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:09,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:07:09,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:14,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:18,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:19,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:07:20,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:23,628 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:07:24,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:25,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:29,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:07:29,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:30,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:34,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:36,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:37,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:07:40,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:41,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:45,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:46,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:47,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:07:50,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:51,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:53,629 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:07:56,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:07:56,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:07:57,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:01,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:02,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:07,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:07,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:08,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:12,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:14,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:14,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:17,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:17,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:19,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:22,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:23,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:08:24,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:25,905 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:27,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:29,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:32,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:34,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:35,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:37,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:39,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:42,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:44,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:46,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:47,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:49,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:52,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:08:53,630 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:08:55,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:08:56,113 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:08:58,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:01,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:01,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:04,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:09:04,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:06,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:09,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:11,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:14,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:09:14,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:16,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:19,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:22,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:23,631 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:09:24,295 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:09:25,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:27,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:30,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:32,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:33,452 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:09:36,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:37,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:41,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:42,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:43,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:09:46,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:47,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:51,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:53,632 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:09:53,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:09:55,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:09:57,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:09:58,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:02,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:03,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:10:03,896 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:10:03,897 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:10:03,900 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:10:04,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:04,711 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:10:05,716 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:10:07,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:09,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:12,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:14,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:14,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:15,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:10:18,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:19,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:23,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:23,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:10:25,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:25,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:10:28,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:30,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:33,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:35,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:10:36,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:41,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:46,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:46,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:48,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:10:49,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:51,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:53,633 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:10:54,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:10:56,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:10:58,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:10:59,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:01,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:04,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:07,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:08,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:11:10,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:12,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:15,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:18,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:11:18,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:21,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:23,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:23,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:11:26,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:28,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:11:28,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:31,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:34,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:36,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:39,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:40,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:11:41,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:44,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:46,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:50,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:50,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:11:52,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:53,635 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:11:55,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:11:58,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:11:59,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:12:01,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:03,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:06,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:07,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:12:09,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:11,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:14,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:17,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:17,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:19,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:12:19,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:22,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:23,636 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:12:25,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:27,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:29,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:12:30,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:35,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:39,859 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:12:40,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:43,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:45,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:49,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:49,935 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:12:51,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:53,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:12:54,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:57,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:12:59,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:12:59,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:02,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:13:03,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:04,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:08,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:09,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:12,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:13:13,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:14,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:18,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:20,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:22,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:13:23,638 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:13:23,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:25,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:28,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:30,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:33,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:35,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:38,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:40,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:41,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:13:41,883 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:13:41,883 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:13:41,886 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:13:42,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:13:43,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:44,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:13:45,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:48,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:50,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:53,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:13:54,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:13:54,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:13:55,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:13:59,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:01,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:04,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:14:05,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:06,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:10,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:12,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:14,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:14:16,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:17,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:21,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:23,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:23,639 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:14:24,589 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:14:26,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:28,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:31,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:34,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:34,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:14:37,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:39,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:42,310 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:43,733 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:14:44,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:47,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:49,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:53,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:14:53,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:14:53,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:14:55,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:14:58,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:00,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:00,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:03,870 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:15:04,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:05,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:09,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:12,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:14,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:15,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:15:17,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:19,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:23,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:23,640 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:15:25,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:26,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:15:28,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:30,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:33,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:33,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:35,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:36,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:15:38,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:40,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:43,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:46,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:15:46,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:48,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:51,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:53,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:15:54,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:15:56,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:15:56,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:15:59,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:01,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:04,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:06,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:16:06,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:09,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:11,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:14,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:16,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:20,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:22,435 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:16:22,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:23,642 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:16:25,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:27,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:30,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:32,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:16:33,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:35,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:38,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:40,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:42,594 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:16:44,060 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:45,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:49,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:51,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:52,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:16:53,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:16:54,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:16:56,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:16:59,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:02,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:02,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:04,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:04,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:07,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:09,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:12,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:17:12,126 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:17:12,127 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:17:12,129 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:17:12,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:12,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:17:13,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:14,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:15,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:17,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:20,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:23,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:17:23,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:23,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:25,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:30,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:30,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:30,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:30,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:31,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:35,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:36,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:40,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:42,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:42,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:45,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:47,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:51,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:52,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:17:53,643 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:17:54,090 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:17:56,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:17:57,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:02,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:02,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:02,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:06,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:18:07,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:07,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:12,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:13,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:14,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:18:18,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:18,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:23,644 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:18:23,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:23,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:24,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:18:28,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:28,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:33,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:34,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:18:34,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:34,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:38,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:39,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:43,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:44,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:46,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:18:49,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:49,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:53,645 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:18:54,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:18:54,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:18:55,387 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:18:59,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:01,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:04,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:19:04,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:06,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:09,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:12,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:14,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:19:15,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:17,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:20,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:22,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:23,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:19:24,802 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:19:25,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:27,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:30,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:33,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:33,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:34,869 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:19:36,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:38,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:41,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:43,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:44,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:19:47,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:48,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:52,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:19:53,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:19:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:19:55,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:19:57,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:01,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:03,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:05,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:20:06,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:08,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:11,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:12,123 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:20:13,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:16,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:18,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:22,202 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:20:23,646 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:20:23,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:26,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:28,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:33,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:33,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:33,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:33,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:34,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:20:34,703 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:20:34,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:20:34,707 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:20:35,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:20:36,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:20:38,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:38,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:43,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:44,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:46,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:20:49,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:49,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:53,647 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:20:54,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:20:55,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:20:56,428 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:20:59,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:00,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:04,487 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:21:05,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:05,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:10,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:10,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:14,556 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:21:16,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:16,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:21,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:21,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:23,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:21:26,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:26,652 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:21:26,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:31,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:31,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:36,718 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:21:36,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:37,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:37,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:41,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:43,181 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:46,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:21:47,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:48,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:52,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:53,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:21:53,649 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:21:54,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:21:57,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:21:58,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:02,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:04,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:04,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:22:08,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:09,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:14,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:14,978 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:22:15,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:19,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:20,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:23,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:22:24,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:25,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:30,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:30,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:31,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:22:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:35,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:41,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:22:41,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:41,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:46,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:46,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:51,214 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:22:51,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:51,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:22:53,650 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:22:56,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:22:57,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:01,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:01,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:03,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:07,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:07,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:08,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:08,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:12,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:13,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:18,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:19,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:20,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:23,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:23,651 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:23:24,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:28,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:28,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:30,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:33,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:35,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:38,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:39,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:41,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:44,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:46,566 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:46,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:49,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:51,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:53,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:23:54,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:23:54,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:23:54,893 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:23:54,894 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:23:54,896 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:23:55,623 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:23:56,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:23:56,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:23:59,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:01,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:05,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:05,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:06,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:24:07,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:10,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:12,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:15,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:16,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:24:17,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:20,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:22,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:23,652 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:24:25,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:26,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:24:28,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:30,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:33,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:34,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:24:36,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:38,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:41,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:44,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:44,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:24:46,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:49,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:52,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:52,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:53,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:24:54,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:24:55,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:24:57,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:24:59,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:02,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:05,079 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:25:05,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:07,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:10,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:13,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:15,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:25:15,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:18,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:20,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:23,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:25:23,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:23,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:23,653 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:25:25,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:28,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:30,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:33,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:25:34,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:35,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:39,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:40,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:45,341 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:25:45,399 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:46,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:50,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:50,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:50,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:51,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:25:53,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:25:53,654 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:25:55,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:25:56,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:00,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:02,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:03,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:05,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:07,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:11,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:12,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:13,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:16,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:17,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:20,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:21,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:22,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:23,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:26:28,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:28,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:30,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:33,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:34,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:38,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:38,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:39,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:40,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:43,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:44,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:48,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:49,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:50,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:53,655 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:26:53,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:26:55,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:26:58,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:26:58,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:01,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:03,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:06,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:07,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:27:07,145 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:27:07,146 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:27:07,149 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:27:07,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:27:08,887 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:27:09,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:09,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:11,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:15,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:16,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:16,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:27:20,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:21,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:23,656 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:27:25,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:26,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:29,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:27:30,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:31,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:36,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:36,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:36,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:39,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:27:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:41,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:47,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:47,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:49,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:27:52,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:52,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:27:53,657 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:27:57,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:27:57,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:27:57,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:02,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:03,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:07,279 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:28:08,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:08,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:13,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:14,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:17,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:28:18,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:19,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:23,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:28:24,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:24,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:24,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:27,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:28:29,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:29,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:34,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:35,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:37,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:28:40,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:40,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:45,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:45,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:45,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:28:50,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:50,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:28:53,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:28:55,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:28:55,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:55,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:28:55,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:00,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:01,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:05,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:29:06,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:06,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:11,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:12,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:15,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:29:16,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:17,524 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:21,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:22,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:23,659 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:29:27,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:28,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:28,825 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:29:32,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:33,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:37,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:38,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:38,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:29:42,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:44,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:46,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:29:47,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:49,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:52,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:29:53,661 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:29:54,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:54,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:29:57,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:29:58,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:00,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:03,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:05,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:07,099 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:30:08,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:10,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:13,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:15,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:30:15,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:18,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:20,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:23,663 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:30:24,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:25,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:30:25,639 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:30:25,640 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:30:25,642 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:30:26,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:26,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:30:27,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:30:30,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:31,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:35,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:37,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:37,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:30:40,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:43,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:46,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:47,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:30:48,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:51,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:53,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:30:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:54,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:30:56,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:30:57,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:30:59,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:01,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:04,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:07,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:07,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:31:09,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:12,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:15,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:17,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:17,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:31:20,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:22,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:23,665 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:31:25,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:25,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:27,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:27,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:31:30,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:32,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:35,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:37,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:31:37,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:40,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:42,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:46,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:47,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:31:48,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:51,087 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:53,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:31:53,667 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:31:56,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:31:57,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:31:59,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:02,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:04,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:07,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:32:07,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:09,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:13,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:14,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:15,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:32:18,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:19,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:23,667 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:32:23,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:23,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:24,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:25,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:32:28,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:29,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:34,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:35,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:35,186 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:32:39,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:40,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:44,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:45,259 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:32:45,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:49,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:50,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:32:53,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:32:55,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:32:55,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:32:56,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:00,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:01,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:05,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:33:06,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:06,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:11,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:11,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:11,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:15,442 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:33:16,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:16,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:21,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:22,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:23,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:33:23,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:33:27,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:27,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:32,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:32,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:35,590 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:33:37,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:38,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:42,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:33:42,999 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:33:43,000 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:33:43,002 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:33:43,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:43,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:33:44,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:45,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:33:48,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:49,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:53,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:53,668 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:33:53,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:33:54,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:33:58,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:33:59,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:03,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:03,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:34:05,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:08,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:10,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:13,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:13,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:34:15,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:18,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:20,835 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:23,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:34:24,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:25,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:34:26,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:29,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:31,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:33,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:34:34,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:37,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:40,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:42,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:44,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:34:45,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:47,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:52,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:34:53,669 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:34:54,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:34:55,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:34:58,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:01,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:02,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:35:04,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:06,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:09,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:11,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:12,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:35:14,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:16,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:19,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:21,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:35:22,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:23,670 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:35:25,671 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:28,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:30,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:31,360 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:35:33,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:35,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:38,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:38,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:39,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:35:40,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:43,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:45,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:49,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:49,483 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:35:51,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:53,671 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:35:54,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:35:57,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:35:59,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:01,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:36:03,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:04,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:08,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:09,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:11,622 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:36:14,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:14,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:19,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:20,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:21,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:36:23,671 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:36:24,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:25,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:30,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:31,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:31,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:36:35,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:36,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:40,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:41,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:45,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:46,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:50,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:51,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:53,672 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:36:56,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:56,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:36:56,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:36:59,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:01,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:02,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:06,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:07,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:11,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:11,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:12,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:16,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:17,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:19,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:37:19,824 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:37:19,824 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:37:19,827 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:37:20,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:37:21,054 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:21,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:22,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:22,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:23,672 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:37:27,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:27,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:28,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:32,117 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:32,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:34,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:38,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:39,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:40,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:43,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:44,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:48,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:49,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:50,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:37:53,673 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:37:53,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:37:55,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:37:59,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:00,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:00,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:04,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:05,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:09,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:10,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:11,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:16,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:16,054 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:18,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:21,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:22,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:23,675 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:38:26,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:26,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:27,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:28,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:31,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:32,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:36,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:37,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:37,542 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:41,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:42,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:47,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:47,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:47,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:52,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:38:52,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:53,676 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:38:57,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:38:57,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:38:58,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:02,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:03,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:07,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:39:08,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:08,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:13,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:14,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:14,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:17,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:39:19,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:19,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:23,677 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:39:24,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:25,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:27,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:39:30,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:30,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:35,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:35,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:37,937 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:39:40,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:40,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:45,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:46,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:47,995 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:39:51,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:51,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:53,678 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:39:56,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:39:57,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:39:58,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:01,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:02,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:07,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:07,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:08,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:12,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:13,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:13,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:18,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:18,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:18,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:23,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:23,680 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:40:24,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:26,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:29,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:29,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:34,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:34,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:35,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:40:35,112 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:40:35,113 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:40:35,115 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:40:35,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:40:35,329 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:36,332 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:39,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:40,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:40,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:45,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:45,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:48,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:40:50,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:50,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:53,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:40:55,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:55,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:40:55,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:40:58,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:00,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:00,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:05,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:05,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:06,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:10,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:10,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:15,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:16,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:16,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:21,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:22,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:23,681 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:41:24,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:26,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:27,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:31,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:32,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:34,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:37,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:37,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:42,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:42,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:43,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:47,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:47,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:51,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:41:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:53,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:53,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:41:58,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:41:59,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:41:59,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:01,908 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:42:04,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:04,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:09,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:09,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:11,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:42:14,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:14,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:19,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:19,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:22,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:42:23,682 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:42:24,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:25,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:30,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:30,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:32,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:42:35,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:35,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:40,303 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:42:40,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:41,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:46,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:46,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:50,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:42:51,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:52,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:53,683 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:42:56,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:42:57,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:42:57,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:00,439 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:01,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:02,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:06,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:07,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:10,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:12,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:12,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:17,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:18,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:18,575 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:22,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:23,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:23,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:43:27,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:28,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:28,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:32,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:33,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:37,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:38,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:38,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:42,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:44,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:44,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:43:44,782 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:43:44,783 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:43:44,785 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:43:45,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:43:45,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:46,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:48,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:49,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:53,684 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:43:54,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:54,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:43:56,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:43:59,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:43:59,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:05,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:05,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:06,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:44:10,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:10,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:15,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:15,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:15,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:44:20,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:22,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:23,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:44:24,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:44:26,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:27,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:31,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:32,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:34,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:44:37,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:37,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:42,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:42,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:42,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:44,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:44:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:48,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:53,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:53,685 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:44:53,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:44:54,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:44:58,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:44:58,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:03,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:04,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:45:04,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:08,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:09,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:13,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:14,356 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:45:14,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:19,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:20,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:22,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:45:23,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:45:25,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:25,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:30,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:30,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:30,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:34,502 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:45:35,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:35,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:40,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:41,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:42,563 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:45:46,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:46,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:51,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:51,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:45:52,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:45:53,686 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:45:56,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:45:56,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:01,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:01,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:02,707 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:46:06,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:07,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:10,764 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:46:12,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:12,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:17,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:18,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:20,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:46:22,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:23,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:23,687 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:46:27,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:28,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:30,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:46:33,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:33,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:38,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:39,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:40,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:46:43,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:44,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:49,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:50,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:51,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:46:53,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:46:54,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:46:56,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:46:58,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:46:58,136 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:46:58,137 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:46:58,140 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:46:59,100 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:47:00,106 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:00,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:01,111 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:01,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:05,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:06,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:08,161 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:10,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:11,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:15,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:17,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:18,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:21,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:22,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:23,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:47:26,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:27,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:28,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:31,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:32,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:36,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:36,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:37,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:41,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:43,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:43,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:46,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:47,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:48,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:52,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:53,688 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:47:54,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:47:56,457 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:47:58,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:47:59,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:03,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:04,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:06,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:48:08,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:09,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:13,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:15,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:16,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:48:19,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:20,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:23,690 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:48:24,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:48:24,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:25,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:29,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:31,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:34,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:48:35,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:36,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:40,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:42,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:44,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:48:45,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:47,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:50,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:52,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:48:52,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:48:53,691 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:48:56,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:48:57,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:01,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:03,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:04,914 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:49:07,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:08,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:12,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:12,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:49:14,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:17,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:19,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:23,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:49:23,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:23,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:49:24,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:28,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:30,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:30,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:33,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:49:34,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:39,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:40,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:41,162 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:49:44,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:45,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:50,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:51,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:49:51,237 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:49:53,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:49:55,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:49:56,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:01,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:01,297 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:50:01,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:06,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:06,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:08,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:50:08,409 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:50:08,409 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:50:08,412 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:50:09,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:50:10,359 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:50:11,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:12,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:17,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:17,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:17,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:18,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:50:22,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:22,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:23,692 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:50:27,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:28,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:50:32,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:33,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:37,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:38,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:50:39,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:42,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:44,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:48,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:48,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:50:50,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:53,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:53,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:50:55,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:50:58,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:50:58,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:51:01,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:03,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:06,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:09,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:11,049 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:12,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:51:14,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:16,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:20,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:21,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:22,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:51:23,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:51:25,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:26,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:30,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:30,895 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:51:32,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:35,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:37,333 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:40,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:40,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:51:42,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:45,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:47,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:51,031 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:51:51,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:52,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:53,694 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:51:56,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:51:57,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:51:59,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:01,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:02,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:06,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:07,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:09,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:12,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:12,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:17,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:17,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:18,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:22,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:23,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:23,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:52:27,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:27,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:28,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:32,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:33,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:37,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:37,525 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:39,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:43,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:44,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:47,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:48,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:49,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:53,696 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:52:53,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:52:55,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:52:55,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:52:58,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:00,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:04,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:05,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:05,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:09,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:10,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:12,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:15,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:15,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:20,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:20,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:21,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:53:21,919 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:53:21,920 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:53:21,923 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:53:22,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:53:22,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:23,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:53:24,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:25,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:25,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:30,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:31,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:32,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:36,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:36,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:41,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:41,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:42,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:46,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:46,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:50,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:53:51,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:53:51,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:53,697 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:53:57,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:53:57,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:00,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:54:02,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:02,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:07,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:08,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:08,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:54:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:13,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:18,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:19,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:54:19,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:23,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:54:24,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:25,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:27,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:54:30,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:30,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:35,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:35,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:35,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:37,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:54:40,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:40,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:45,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:45,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:47,206 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:54:50,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:50,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:53,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:54:55,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:54:55,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:54:57,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:00,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:00,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:05,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:06,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:07,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:10,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:11,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:15,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:16,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:17,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:22,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:22,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:23,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:55:25,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:27,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:27,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:32,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:33,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:35,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:38,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:38,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:43,582 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:44,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:44,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:49,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:50,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:53,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:55:53,701 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:55:54,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:55:55,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:55:59,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:00,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:01,710 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:05,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:05,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:10,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:10,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:11,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:15,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:15,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:20,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:20,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:20,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:23,702 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:56:26,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:27,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:28,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:56:28,624 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:56:28,624 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:56:28,627 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:56:28,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:56:28,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:30,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:32,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:32,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:37,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:37,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:37,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:38,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:42,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:42,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:47,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:48,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:49,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:52,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:53,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:56:53,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:56:57,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:56:58,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:56:58,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:03,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:04,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:04,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:07,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:57:08,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:09,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:13,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:14,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:17,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:57:18,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:19,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:23,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:57:24,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:25,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:25,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:57:29,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:30,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:34,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:35,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:57:35,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:39,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:43,370 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:57:44,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:46,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:49,662 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:51,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:53,431 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:57:53,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:57:54,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:57:56,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:57:59,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:01,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:58:01,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:05,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:06,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:10,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:11,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:58:12,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:15,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:18,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:18,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:21,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:21,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:58:23,704 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:58:23,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:26,706 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:29,112 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:31,664 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:58:32,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:34,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:37,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:39,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:39,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:58:42,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:44,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:47,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:49,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:49,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:58:52,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:53,705 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:58:55,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:58:57,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:58:59,847 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:00,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:02,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:06,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:07,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:08,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:11,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:13,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:16,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:17,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:18,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:21,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:23,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:23,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:59:25,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:26,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:28,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:32,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:33,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:34,040 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 00:59:34,041 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 00:59:34,042 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 00:59:34,044 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 00:59:34,078 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 00:59:35,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:37,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:39,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:43,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:44,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:45,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:48,098 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:49,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:53,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:53,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:53,706 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 00:59:54,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 00:59:55,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 00:59:58,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 00:59:59,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:03,289 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:00:03,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:03,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:04,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:08,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:09,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:13,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:00:14,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:15,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:20,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:20,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:20,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:23,475 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:00:23,707 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:00:25,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:25,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:30,318 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:31,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:31,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:00:35,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:36,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:40,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:41,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:00:41,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:45,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:46,893 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:51,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:51,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:00:52,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:00:53,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:00:56,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:00:57,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:01,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:01,725 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:01:03,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:07,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:07,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:08,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:09,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:01:12,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:13,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:17,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:18,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:19,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:01:22,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:23,708 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:01:24,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:27,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:01:28,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:30,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:35,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:35,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:37,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:01:40,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:40,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:45,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:45,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:48,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:01:50,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:50,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:53,709 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:01:55,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:01:55,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:01:56,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:00,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:00,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:05,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:06,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:06,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:11,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:11,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:14,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:16,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:17,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:22,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:22,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:22,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:23,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:23,710 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:02:27,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:27,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:32,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:32,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:33,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:37,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:37,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:40,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:02:40,922 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:02:40,923 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:02:40,926 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:02:41,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:02:41,382 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:42,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:43,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:43,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:47,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:48,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:51,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:02:52,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:53,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:02:53,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:02:58,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:02:59,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:01,503 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:03:04,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:04,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:09,562 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:03:09,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:09,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:14,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:14,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:19,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:03:20,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:20,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:23,711 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:03:25,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:26,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:29,699 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:03:31,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:31,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:36,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:36,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:37,757 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:03:41,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:42,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:47,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:47,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:47,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:03:52,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:52,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:53,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:03:57,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:03:57,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:03:57,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:02,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:03,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:05,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:08,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:08,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:13,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:14,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:16,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:19,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:19,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:23,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:04:24,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:24,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:26,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:29,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:29,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:34,152 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:34,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:35,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:40,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:40,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:40,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:44,225 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:45,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:46,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:51,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:04:51,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:53,713 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:04:54,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:04:56,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:04:56,714 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:01,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:01,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:06,358 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:05:06,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:06,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:11,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:13,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:16,424 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:05:17,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:18,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:22,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:23,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:23,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:23,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:05:25,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:05:27,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:28,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:32,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:33,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:05:34,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:38,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:39,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:43,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:43,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:05:44,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:48,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:50,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:51,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:05:51,753 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:05:51,754 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:05:51,756 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:05:52,653 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:05:53,659 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:05:53,714 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:05:53,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:05:55,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:05:58,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:01,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:01,709 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:04,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:06,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:09,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:11,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:11,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:11,779 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:14,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:16,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:19,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:21,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:21,850 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:23,715 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:06:24,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:27,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:29,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:29,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:32,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:34,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:37,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:39,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:40,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:42,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:45,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:48,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:50,046 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:50,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:53,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:06:53,716 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:06:55,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:06:58,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:06:58,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:00,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:03,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:05,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:08,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:07:08,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:11,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:14,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:16,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:18,215 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:07:19,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:21,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:23,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:07:24,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:24,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:26,266 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:07:26,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:29,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:31,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:34,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:36,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:07:37,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:39,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:42,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:44,381 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:07:44,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:48,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:49,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:53,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:07:53,717 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:07:54,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:07:58,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:00,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:04,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:04,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:08:05,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:09,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:11,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:11,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:14,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:08:14,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:19,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:21,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:22,620 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:08:23,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:08:25,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:26,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:31,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:31,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:32,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:08:36,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:36,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:41,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:41,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:08:42,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:46,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:47,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:49,788 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:08:51,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:52,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:53,718 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:08:57,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:08:58,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:08:59,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:03,400 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:03,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:07,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:09:07,859 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:09:07,877 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:09:07,900 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:09:07,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:09:08,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:08,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:08,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:09,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:13,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:13,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:17,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:19,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:19,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:23,719 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:09:24,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:25,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:25,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:28,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:30,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:30,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:35,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:35,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:36,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:40,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:40,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:46,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:46,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:46,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:51,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:52,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:09:53,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:09:56,230 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:09:56,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:09:57,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:01,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:03,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:06,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:10:07,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:08,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:12,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:14,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:14,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:14,365 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:10:18,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:19,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:23,720 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:10:23,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:24,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:10:24,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:28,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:30,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:32,484 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:10:33,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:35,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:38,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:40,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:42,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:10:43,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:45,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:48,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:50,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:10:51,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:53,722 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:10:54,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:10:56,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:10:59,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:00,661 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:01,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:04,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:07,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:10,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:10,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:12,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:15,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:17,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:18,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:20,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:22,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:23,723 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:11:25,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:28,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:28,852 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:30,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:33,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:35,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:38,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:38,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:41,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:43,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:47,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:47,975 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:49,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:52,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:53,725 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:11:54,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:11:58,043 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:11:58,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:11:59,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:03,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:05,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:08,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:08,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:10,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:13,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:15,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:12:15,392 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:12:15,393 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:12:15,395 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:12:16,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:16,156 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:12:16,158 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:18,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:19,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:21,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:23,726 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:12:24,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:26,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:26,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:29,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:31,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:34,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:37,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:38,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:40,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:42,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:45,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:48,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:48,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:51,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:53,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:12:54,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:12:57,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:12:58,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:12:59,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:02,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:05,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:06,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:13:07,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:10,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:12,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:15,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:16,588 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:13:17,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:20,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:22,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:23,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:13:24,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:13:25,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:27,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:30,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:32,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:34,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:13:35,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:37,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:41,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:41,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:43,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:44,778 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:13:46,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:48,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:52,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:52,826 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:13:53,727 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:13:54,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:13:57,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:13:59,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:02,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:02,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:14:05,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:07,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:10,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:12,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:15,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:17,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:20,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:22,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:23,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:14:25,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:27,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:30,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:32,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:36,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:37,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:14:37,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:41,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:43,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:43,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:47,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:47,145 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:14:48,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:52,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:53,728 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:14:54,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:14:54,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:14:57,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:14:59,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:02,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:04,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:04,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:08,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:09,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:14,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:14,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:15,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:15,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:19,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:20,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:23,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:15:24,388 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:24,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:25,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:29,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:30,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:30,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:32,443 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:35,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:35,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:40,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:41,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:42,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:45,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:46,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:49,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:15:49,588 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:15:49,588 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:15:49,590 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:15:49,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:15:50,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:50,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:51,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:15:51,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:52,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:15:53,729 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:15:55,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:15:56,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:00,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:01,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:05,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:06,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:10,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:11,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:15,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:16,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:20,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:21,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:23,730 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:16:26,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:26,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:31,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:31,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:36,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:36,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:41,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:41,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:46,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:46,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:51,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:51,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:16:53,732 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:16:56,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:16:57,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:01,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:02,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:06,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:07,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:11,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:12,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:16,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:17,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:21,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:22,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:23,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:17:26,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:28,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:31,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:33,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:36,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:38,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:41,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:43,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:46,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:48,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:51,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:53,733 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:17:54,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:17:56,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:17:59,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:01,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:04,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:06,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:09,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:11,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:14,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:16,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:19,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:21,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:23,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:18:25,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:26,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:30,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:31,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:35,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:36,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:40,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:41,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:45,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:46,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:50,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:18:51,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:53,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:18:56,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:18:56,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:01,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:01,735 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:06,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:06,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:11,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:11,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:16,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:16,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:21,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:21,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:23,734 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:19:26,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:27,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:31,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:32,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:36,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:37,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:41,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:42,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:46,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:47,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:51,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:52,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:19:53,735 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:19:56,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:19:58,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:01,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:03,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:06,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:08,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:11,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:13,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:16,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:18,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:21,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:23,735 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:20:24,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:26,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:29,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:31,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:34,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:36,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:39,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:41,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:44,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:46,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:49,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:51,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:20:53,735 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:20:55,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:20:56,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:00,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:01,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:05,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:06,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:10,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:11,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:15,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:16,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:20,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:21,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:23,736 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:21:26,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:26,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:31,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:31,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:36,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:36,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:41,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:41,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:46,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:46,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:51,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:51,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:21:53,736 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:21:56,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:21:57,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:01,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:02,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:06,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:07,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:11,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:12,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:16,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:17,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:21,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:22,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:23,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:22:26,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:28,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:31,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:33,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:36,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:38,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:41,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:43,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:46,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:48,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:51,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:53,737 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:22:54,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:22:56,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:22:59,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:01,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:04,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:06,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:09,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:11,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:14,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:16,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:19,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:21,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:23,738 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:23:25,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:26,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:30,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:31,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:35,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:36,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:40,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:41,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:45,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:46,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:50,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:51,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:23:53,738 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:23:55,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:23:56,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:00,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:01,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:05,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:06,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:10,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:11,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:15,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:16,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:20,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:21,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:23,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:24:26,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:26,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:31,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:31,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:36,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:36,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:41,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:41,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:46,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:46,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:51,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:51,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:24:53,739 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:24:56,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:24:57,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:01,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:02,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:06,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:07,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:11,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:12,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:16,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:17,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:21,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:22,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:23,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:25:26,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:28,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:31,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:33,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:36,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:38,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:41,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:25:41,411 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:25:41,412 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:25:41,415 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:25:41,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:42,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:25:42,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:25:43,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:25:44,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:46,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:49,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:51,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:53,740 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:25:54,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:25:56,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:25:59,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:01,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:04,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:06,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:09,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:11,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:13,504 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:26:15,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:17,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:20,681 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:22,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:23,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:26:25,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:27,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:30,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:33,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:35,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:38,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:40,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:44,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:45,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:50,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:51,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:52,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:26:53,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:26:53,747 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:26:56,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:26:56,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:56,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:56,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:26:58,772 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:27:01,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:03,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:06,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:11,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:16,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:16,146 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:17,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:18,863 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:27:21,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:22,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:23,741 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:27:26,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:27,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:31,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:32,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:36,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:37,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:41,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:42,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:46,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:47,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:51,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:52,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:27:53,742 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:27:56,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:27:58,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:01,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:03,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:06,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:08,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:11,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:13,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:16,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:28:16,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:19,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:21,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:23,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:28:24,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:26,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:27,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:28:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:31,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:34,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:36,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:37,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:28:40,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:43,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:45,306 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:28:45,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:48,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:50,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:53,743 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:28:53,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:28:55,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:28:56,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:28:58,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:01,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:04,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:04,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:05,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:29:06,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:09,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:11,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:15,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:15,511 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:29:16,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:20,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:21,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:23,744 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:29:25,564 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:29:25,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:27,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:30,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:32,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:35,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:29:36,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:36,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:37,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:41,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:42,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:45,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:29:46,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:47,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:52,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:53,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:29:53,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:29:53,759 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:29:57,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:29:58,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:02,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:02,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:03,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:30:04,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:07,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:09,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:13,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:13,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:30:15,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:19,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:19,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:20,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:23,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:30:23,964 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:30:24,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:25,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:30,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:31,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:32,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:30:35,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:36,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:40,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:41,590 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:42,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:30:45,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:46,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:51,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:51,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:52,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:30:52,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:30:53,745 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:30:56,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:30:57,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:01,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:02,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:31:02,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:06,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:07,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:11,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:12,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:31:13,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:17,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:18,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:20,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:31:22,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:23,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:23,746 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:31:28,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:31:28,038 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:31:28,038 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:31:28,041 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:31:28,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:28,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:31:29,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:29,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:31:33,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:34,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:38,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:39,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:39,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:31:43,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:44,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:47,533 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:31:48,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:49,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:53,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:31:53,747 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:31:55,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:31:58,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:00,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:03,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:32:04,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:05,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:09,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:10,777 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:13,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:32:14,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:16,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:20,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:20,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:21,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:23,747 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:32:23,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:32:25,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:26,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:30,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:31,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:35,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:36,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:40,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:41,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:45,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:46,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:50,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:51,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:53,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:32:56,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:32:57,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:32:59,969 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:33:01,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:02,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:06,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:08,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:08,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:33:11,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:13,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:17,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:18,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:20,119 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:33:22,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:23,749 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:33:24,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:28,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:30,192 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:33:30,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:33,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:35,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:38,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:40,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:33:40,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:43,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:45,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:49,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:50,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:33:51,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:53,751 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:33:55,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:33:56,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:33:58,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:00,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:01,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:06,096 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:06,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:06,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:08,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:11,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:11,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:16,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:17,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:18,527 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:21,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:22,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:23,752 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:34:26,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:27,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:28,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:31,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:32,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:37,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:37,681 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:38,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:42,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:43,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:45,737 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:47,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:49,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:53,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:53,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:53,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:34:54,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:34:55,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:34:58,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:34:59,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:03,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:04,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:05,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:08,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:09,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:12,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:35:12,928 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:35:12,929 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:35:12,931 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:35:13,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:13,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:35:13,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:14,926 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:14,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:18,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:19,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:23,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:35:23,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:24,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:25,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:30,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:30,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:34,067 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:35,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:36,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:41,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:41,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:44,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:46,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:46,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:51,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:51,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:35:52,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:35:53,754 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:35:56,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:35:57,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:02,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:36:02,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:03,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:08,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:08,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:08,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:12,345 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:36:13,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:13,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:18,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:19,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:22,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:36:23,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:36:24,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:24,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:29,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:30,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:30,541 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:36:34,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:35,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:40,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:40,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:36:40,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:45,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:46,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:50,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:36:51,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:51,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:36:53,755 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:36:56,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:36:57,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:00,752 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:37:02,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:02,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:07,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:07,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:10,806 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:37:12,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:13,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:18,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:18,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:19,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:37:23,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:23,756 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:37:24,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:29,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:29,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:37:30,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:35,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:35,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:35,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:38,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:37:40,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:41,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:46,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:46,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:48,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:37:51,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:51,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:53,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:37:57,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:37:57,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:37:58,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:02,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:03,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:06,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:07,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:08,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:12,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:13,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:16,273 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:18,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:18,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:23,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:23,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:38:23,606 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:38:23,607 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:38:23,609 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:38:23,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:38:24,331 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:38:24,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:24,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:25,337 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:26,340 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:28,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:29,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:34,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:34,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:35,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:39,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:41,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:44,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:44,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:46,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:49,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:51,688 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:53,757 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:38:54,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:38:54,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:38:56,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:38:59,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:02,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:02,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:39:05,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:07,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:10,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:12,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:12,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:39:15,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:17,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:20,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:22,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:22,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:39:23,758 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:39:26,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:27,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:32,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:32,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:39:33,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:37,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:39,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:39,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:42,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:42,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:39:44,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:47,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:49,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:52,906 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:39:53,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:39:53,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:39:54,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:39:58,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:00,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:02,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:04,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:05,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:10,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:10,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:10,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:15,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:15,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:20,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:20,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:21,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:23,759 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:40:26,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:27,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:30,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:32,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:32,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:37,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:37,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:37,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:38,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:42,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:42,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:47,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:47,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:48,280 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:52,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:52,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:40:53,760 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:40:58,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:40:58,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:40:58,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:03,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:04,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:08,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:09,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:09,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:14,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:15,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:16,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:19,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:20,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:23,763 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:41:24,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:25,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:26,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:30,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:30,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:34,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:41:34,240 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:41:34,240 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:41:34,242 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:41:34,576 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:41:35,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:35,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:36,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:36,586 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:40,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:42,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:44,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:45,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:47,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:50,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:53,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:53,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:41:53,762 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:41:54,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:41:55,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:41:58,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:00,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:03,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:05,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:08,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:10,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:13,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:15,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:18,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:20,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:23,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:23,763 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:42:26,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:28,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:30,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:42:32,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:33,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:37,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:39,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:40,974 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:42:43,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:44,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:48,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:49,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:53,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:42:53,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:53,763 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:42:54,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:42:58,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:42:59,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:01,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:04,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:04,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:10,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:10,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:10,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:11,177 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:15,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:15,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:20,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:21,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:21,256 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:23,763 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:43:26,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:27,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:27,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:30,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:32,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:32,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:37,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:37,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:38,368 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:42,732 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:42,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:47,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:48,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:48,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:52,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:53,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:43:53,765 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:43:58,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:43:58,505 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:43:58,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:03,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:03,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:08,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:44:08,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:08,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:13,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:14,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:18,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:44:19,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:19,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:23,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:44:24,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:25,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:26,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:44:30,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:30,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:35,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:35,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:36,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:44:40,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:40,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:44,819 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:44:45,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:45,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:50,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:50,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:44:53,766 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:44:54,972 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:44:56,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:44:56,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:01,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:02,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:05,037 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:07,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:07,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:12,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:12,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:12,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:12,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:45:12,303 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:45:12,304 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:45:12,306 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:45:13,085 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:45:14,092 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:15,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:17,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:17,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:22,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:22,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:23,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:23,767 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:45:27,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:28,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:33,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:33,229 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:34,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:39,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:39,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:39,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:43,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:44,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:44,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:49,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:45:50,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:51,348 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:45:53,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:45:55,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:45:55,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:00,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:01,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:01,419 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:46:05,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:06,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:10,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:10,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:46:11,473 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:15,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:16,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:20,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:46:21,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:22,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:23,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:46:26,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:26,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:27,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:28,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:46:31,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:32,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:37,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:38,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:38,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:46:42,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:43,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:47,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:48,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:48,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:46:52,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:53,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:46:53,768 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:46:57,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:46:58,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:02,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:02,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:47:04,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:08,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:09,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:10,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:47:13,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:14,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:19,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:20,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:20,956 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:47:23,769 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:47:24,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:25,771 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:29,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:30,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:31,032 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:47:34,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:35,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:39,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:47:40,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:41,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:46,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:46,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:49,141 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:47:51,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:51,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:53,770 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:47:57,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:47:57,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:47:57,195 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:02,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:02,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:07,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:07,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:07,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:12,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:12,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:17,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:18,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:18,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:23,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:23,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:23,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:48:24,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:48:24,528 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:48:24,529 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:48:24,531 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:48:25,434 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:48:26,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:27,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:28,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:28,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:33,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:33,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:35,498 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:38,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:38,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:44,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:44,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:44,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:45,557 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:49,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:49,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:53,771 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:48:54,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:54,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:48:55,629 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:48:59,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:48:59,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:04,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:05,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:49:05,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:10,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:10,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:10,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:10,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:12,739 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:49:15,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:15,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:20,963 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:21,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:22,803 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:49:23,772 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:49:26,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:26,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:31,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:32,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:32,882 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:49:36,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:37,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:40,931 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:49:41,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:43,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:46,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:48,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:51,001 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:49:52,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:53,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:49:53,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:49:58,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:58,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:49:58,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:01,062 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:03,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:04,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:08,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:09,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:11,135 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:13,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:14,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:18,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:19,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:19,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:23,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:23,773 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:50:25,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:25,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:26,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:28,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:30,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:34,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:35,292 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:36,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:39,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:41,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:45,045 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:45,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:47,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:51,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:52,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:50:53,433 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:50:53,774 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:50:56,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:50:57,775 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:01,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:01,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:02,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:03,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:06,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:07,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:11,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:12,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:13,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:16,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:17,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:21,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:22,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:23,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:23,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:51:28,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:28,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:28,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:29,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:51:29,828 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:51:29,829 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:51:29,831 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:51:30,693 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:51:31,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:32,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:33,656 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:33,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:38,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:38,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:41,808 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:43,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:43,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:49,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:49,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:49,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:53,775 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:51:54,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:51:54,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:51:58,915 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:51:59,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:00,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:05,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:05,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:08,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:52:10,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:10,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:15,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:16,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:16,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:17,033 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:52:21,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:21,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:23,777 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:52:26,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:26,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:27,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:52:31,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:31,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:36,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:37,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:52:37,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:41,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:42,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:47,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:52:47,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:47,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:52,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:52,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:52:53,777 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:52:55,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:52:57,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:52:57,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:02,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:03,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:05,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:53:08,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:08,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:13,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:13,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:15,418 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:53:18,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:18,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:23,497 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:53:23,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:53:24,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:24,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:29,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:30,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:33,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:53:35,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:35,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:40,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:40,461 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:41,616 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:53:45,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:45,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:51,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:51,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:51,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:53:53,778 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:53:56,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:53:57,744 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:53:57,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:01,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:01,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:02,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:06,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:08,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:09,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:12,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:13,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:17,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:18,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:19,888 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:22,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:23,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:23,779 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:54:28,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:28,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:29,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:33,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:34,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:36,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:54:36,802 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:54:36,802 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:54:36,804 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:54:37,012 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:54:38,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:38,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:39,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:39,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:43,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:44,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:44,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:48,088 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:54:49,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:49,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:53,780 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:54:54,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:54:55,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:54:57,147 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:00,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:00,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:05,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:05,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:05,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:10,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:10,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:15,270 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:16,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:16,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:21,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:22,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:23,780 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:55:25,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:26,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:27,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:31,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:32,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:32,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:35,401 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:36,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:37,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:42,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:43,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:43,481 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:47,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:48,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:52,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:55:53,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:55:53,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:53,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:55:58,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:55:58,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:03,618 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:56:04,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:04,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:09,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:09,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:11,674 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:56:14,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:14,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:19,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:20,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:21,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:56:23,781 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:56:24,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:25,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:29,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:30,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:30,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:31,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:56:35,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:35,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:40,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:41,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:41,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:56:45,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:46,454 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:51,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:51,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:56:52,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:56:53,782 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:56:56,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:56:57,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:00,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:02,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:02,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:07,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:07,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:10,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:12,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:13,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:18,363 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:18,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:20,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:23,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:23,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:23,784 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:57:28,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:28,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:28,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:33,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:34,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:37,362 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:38,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:39,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:43,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:45,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:45,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:45,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 01:57:45,600 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 01:57:45,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 01:57:45,603 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 01:57:46,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 01:57:46,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:47,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:49,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:50,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:53,785 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:57:54,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:57:55,486 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:57:55,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:57:59,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:00,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:05,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:05,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:58:05,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:10,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:12,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:15,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:15,610 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:58:17,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:20,410 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:22,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:23,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:58:23,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:58:25,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:27,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:30,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:32,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:32,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:33,738 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:58:35,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:37,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:40,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:43,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:43,805 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:58:46,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:49,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:51,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:51,868 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:58:53,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:58:54,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:58:57,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:58:59,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:01,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:59:03,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:04,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:08,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:10,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:12,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:59:13,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:15,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:18,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:20,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:59:20,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:23,786 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:59:23,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:27,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:29,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:30,116 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:59:32,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:34,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:37,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:39,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:40,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:59:42,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:44,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:47,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:48,239 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 01:59:50,163 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:52,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 01:59:53,787 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 01:59:55,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 01:59:58,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:00,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:00:01,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:03,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:06,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:08,478 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:10,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:00:11,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:13,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:16,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:18,468 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:00:19,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:22,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:23,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:00:25,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:27,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:28,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:00:30,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:32,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:35,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:35,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:37,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:00:37,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:40,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:42,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:45,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:47,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:51,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:51,677 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:00:53,788 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:00:53,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:58,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:00:58,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:00:59,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:00:59,162 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:00:59,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:00:59,166 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:00:59,730 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:00:59,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:00,736 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:03,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:04,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:08,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:09,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:09,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:13,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:14,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:19,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:19,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:19,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:23,789 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:01:24,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:24,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:29,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:30,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:30,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:35,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:36,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:36,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:37,977 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:40,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:41,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:45,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:46,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:48,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:01:50,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:51,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:53,790 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:01:55,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:01:56,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:01:58,120 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:01,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:01,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:06,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:06,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:07,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:11,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:12,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:16,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:18,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:18,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:21,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:23,791 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:02:23,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:26,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:27,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:28,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:32,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:34,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:34,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:36,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:37,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:39,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:42,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:44,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:46,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:47,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:49,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:52,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:53,792 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:02:54,602 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:02:54,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:02:57,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:02:59,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:03,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:04,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:03:05,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:08,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:10,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:13,790 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:13,889 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:03:15,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:18,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:21,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:21,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:21,945 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:03:23,799 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:03:23,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:26,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:28,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:31,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:32,006 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:03:34,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:36,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:39,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:41,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:42,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:03:44,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:46,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:50,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:03:50,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:52,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:52,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:03:53,799 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:03:55,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:03:57,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:00,187 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:00,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:02,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:05,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:07,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:07,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:08,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:04:08,079 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:04:08,079 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:04:08,082 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:04:08,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:04:09,283 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:10,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:11,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:12,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:16,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:18,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:18,349 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:21,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:23,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:23,801 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:04:26,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:28,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:28,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:31,632 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:34,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:34,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:36,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:36,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:39,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:41,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:45,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:46,540 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:47,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:50,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:52,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:04:53,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:04:55,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:04:56,609 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:04:57,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:00,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:02,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:05,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:06,670 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:05:08,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:12,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:13,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:14,723 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:05:17,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:18,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:22,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:22,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:23,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:23,803 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:05:24,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:05:27,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:28,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:32,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:32,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:05:34,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:37,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:39,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:42,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:42,900 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:05:44,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:47,384 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:49,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:51,967 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:05:52,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:53,804 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:05:54,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:05:58,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:05:59,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:02,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:06:03,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:04,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:08,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:08,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:09,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:10,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:06:13,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:14,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:19,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:19,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:20,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:06:23,804 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:06:25,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:26,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:28,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:06:31,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:31,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:36,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:36,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:38,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:06:41,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:42,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:47,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:47,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:48,369 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:06:52,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:52,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:53,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:06:58,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:06:58,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:06:58,430 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:03,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:04,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:06,491 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:09,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:09,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:14,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:14,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:14,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:07:14,569 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:07:14,569 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:07:14,572 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:07:15,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:07:16,565 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:17,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:19,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:19,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:23,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:07:24,614 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:24,665 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:24,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:29,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:29,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:34,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:34,736 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:35,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:39,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:40,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:44,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:44,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:46,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:51,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:07:51,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:53,806 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:07:54,798 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:07:56,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:07:56,807 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:02,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:02,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:02,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:08:07,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:07,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:12,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:08:13,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:13,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:13,164 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:18,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:18,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:20,955 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:08:23,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:23,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:23,807 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:08:28,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:28,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:28,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:31,016 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:08:33,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:33,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:39,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:39,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:41,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:08:44,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:44,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:49,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:08:49,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:49,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:53,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:08:54,812 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:08:55,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:55,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:55,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:08:59,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:09:00,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:00,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:05,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:05,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:06,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:09:10,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:10,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:15,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:16,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:16,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:09:20,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:21,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:23,809 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:09:26,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:26,411 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:09:26,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:31,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:31,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:36,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:36,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:09:36,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:41,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:41,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:46,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:09:46,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:46,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:51,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:51,996 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:09:53,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:09:56,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:09:57,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:01,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:02,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:04,647 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:10:06,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:08,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:13,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:18,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:18,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:20,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:10:23,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:23,810 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:10:23,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:28,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:28,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:28,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:30,885 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:10:33,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:34,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:38,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:10:38,562 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:10:38,564 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:10:38,568 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:10:38,933 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:10:39,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:39,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:39,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:10:40,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:10:44,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:44,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:49,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:49,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:10:51,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:10:53,811 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:10:54,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:10:55,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:00,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:01,082 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:01,572 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:06,574 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:06,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:09,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:11,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:11,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:16,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:17,094 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:19,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:22,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:22,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:23,814 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:11:27,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:27,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:29,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:32,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:32,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:38,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:38,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:39,330 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:43,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:44,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:47,383 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:48,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:49,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:53,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:11:54,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:11:54,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:11:56,447 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:11:59,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:00,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:05,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:05,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:06,514 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:12:10,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:10,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:14,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:12:15,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:15,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:20,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:21,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:23,815 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:12:24,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:12:25,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:26,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:30,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:32,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:34,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:12:36,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:37,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:41,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:43,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:44,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:12:46,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:48,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:51,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:12:52,813 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:12:53,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:12:54,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:12:57,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:00,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:00,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:02,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:03,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:05,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:08,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:11,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:11,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:12,942 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:14,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:16,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:19,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:21,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:23,015 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:23,816 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:13:24,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:26,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:29,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:31,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:31,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:34,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:36,913 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:40,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:41,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:42,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:45,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:47,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:48,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:13:48,894 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:13:48,895 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:13:48,897 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:13:49,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:13:50,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:50,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:51,207 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:13:52,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:53,817 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:13:56,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:13:58,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:58,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:13:59,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:14:02,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:03,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:07,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:08,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:09,336 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:14:12,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:13,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:18,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:19,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:19,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:14:23,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:23,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:14:25,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:27,459 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:14:28,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:30,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:33,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:35,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:37,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:14:38,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:40,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:43,470 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:45,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:48,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:50,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:53,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:14:53,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:14:55,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:14:58,673 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:14:58,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:00,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:03,752 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:06,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:06,726 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:15:09,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:11,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:14,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:16,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:19,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:21,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:22,835 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:15:23,819 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:15:24,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:26,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:29,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:31,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:32,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:15:35,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:36,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:40,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:15:41,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:42,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:42,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:46,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:48,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:51,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:15:51,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:53,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:53,820 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:15:56,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:15:58,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:15:58,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:01,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:01,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:03,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:06,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:08,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:09,132 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:12,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:13,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:17,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:19,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:19,200 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:23,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:23,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:16:24,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:28,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:29,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:29,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:31,284 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:34,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:34,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:39,334 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:39,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:40,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:44,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:46,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:46,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:49,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:49,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:51,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:53,821 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:16:54,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:16:57,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:16:59,458 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:16:59,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:02,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:04,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:07,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:09,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:17:10,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:12,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:15,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:16,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:17:16,599 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:17:16,600 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:17:16,602 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:17:17,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:17:17,600 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:17:18,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:21,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:23,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:17:24,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:27,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:27,660 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:17:29,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:32,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:34,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:37,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:17:37,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:39,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:42,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:45,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:45,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:17:48,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:50,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:53,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:17:53,822 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:17:55,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:17:55,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:17:58,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:01,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:01,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:02,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:18:04,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:06,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:09,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:12,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:12,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:18:15,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:17,891 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:17,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:20,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:23,038 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:18:23,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:23,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:18:25,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:28,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:30,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:33,103 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:18:33,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:36,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:40,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:41,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:43,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:18:45,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:46,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:50,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:51,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:18:53,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:18:53,823 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:18:55,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:18:57,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:01,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:19:01,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:02,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:06,728 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:07,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:11,357 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:19:12,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:13,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:18,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:18,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:21,425 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:19:23,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:23,734 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:23,824 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:19:28,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:28,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:29,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:19:33,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:33,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:39,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:39,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:39,543 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:19:44,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:44,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:49,601 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:19:49,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:49,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:53,825 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:19:55,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:19:55,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:19:59,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:00,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:00,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:05,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:06,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:07,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:11,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:11,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:16,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:17,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:21,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:21,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:23,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:20:26,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:20:26,075 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:20:26,075 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:20:26,078 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:20:26,862 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:20:27,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:27,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:27,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:32,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:32,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:37,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:37,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:38,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:42,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:43,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:43,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:43,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:45,987 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:47,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:48,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:52,897 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:53,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:20:54,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:20:56,050 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:20:58,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:20:59,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:03,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:04,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:06,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:21:08,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:09,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:14,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:14,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:15,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:21:19,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:19,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:23,827 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:21:24,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:25,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:25,234 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:21:29,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:31,445 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:31,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:33,288 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:21:35,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:36,446 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:40,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:41,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:43,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:21:45,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:46,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:50,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:51,404 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:21:51,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:21:53,828 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:21:56,828 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:21:56,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:01,472 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:02,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:02,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:02,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:07,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:07,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:11,537 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:12,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:12,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:17,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:18,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:18,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:21,638 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:23,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:23,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:23,828 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:22:28,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:28,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:29,687 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:33,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:33,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:38,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:38,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:39,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:43,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:43,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:49,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:49,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:49,843 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:53,829 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:22:54,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:22:54,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:22:59,904 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:22:59,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:00,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:05,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:05,638 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:07,966 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:10,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:10,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:15,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:16,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:16,375 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:18,025 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:21,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:21,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:23,831 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:23:26,729 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:26,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:28,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:31,730 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:31,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:35,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:23:35,270 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:23:35,270 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:23:35,272 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:23:36,159 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:23:36,160 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:37,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:37,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:38,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:42,084 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:42,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:46,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:47,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:47,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:52,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:23:52,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:53,831 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:23:55,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:23:57,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:23:57,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:02,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:03,371 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:24:03,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:07,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:08,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:13,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:13,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:24:14,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:18,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:19,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:23,580 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:24:23,832 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:24:23,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:24,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:29,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:30,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:31,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:24:35,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:35,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:40,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:40,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:41,694 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:24:45,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:45,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:51,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:51,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:51,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:24:51,755 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:24:53,832 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:24:56,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:24:56,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:01,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:01,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:25:02,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:06,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:07,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:11,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:25:12,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:12,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:17,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:17,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:17,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:19,951 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:25:22,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:22,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:23,833 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:25:28,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:28,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:30,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:25:33,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:33,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:38,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:38,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:40,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:25:43,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:43,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:48,144 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:25:48,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:49,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:53,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:25:54,836 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:25:55,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:25:58,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:00,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:00,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:05,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:06,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:06,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:08,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:11,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:11,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:16,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:16,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:16,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:21,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:21,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:23,835 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:26:26,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:26,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:26,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:31,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:32,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:36,473 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:36,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:37,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:41,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:43,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:43,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:26:43,412 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:26:43,413 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:26:43,415 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:26:43,523 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:26:44,531 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:47,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:48,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:52,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:53,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:26:53,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:26:54,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:26:57,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:26:58,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:02,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:03,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:27:03,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:07,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:10,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:11,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:27:13,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:15,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:18,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:20,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:20,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:21,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:27:23,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:23,836 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:27:25,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:28,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:30,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:31,828 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:27:34,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:35,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:39,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:41,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:41,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:27:44,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:48,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:48,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:48,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:49,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:27:50,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:53,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:27:53,837 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:27:55,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:27:58,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:00,017 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:00,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:03,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:05,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:09,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:10,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:11,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:14,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:16,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:19,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:20,139 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:21,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:23,837 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:28:24,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:26,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:28,198 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:30,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:32,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:35,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:35,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:37,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:38,262 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:40,486 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:42,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:45,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:47,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:48,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:50,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:52,914 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:28:53,840 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:28:56,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:28:56,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:28:58,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:01,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:03,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:06,436 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:29:06,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:06,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:09,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:11,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:14,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:17,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:19,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:20,513 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:29:22,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:23,849 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:29:25,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:28,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:30,579 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:29:31,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:33,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:36,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:38,637 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:29:38,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:38,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:41,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:43,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:46,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:48,705 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:29:48,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:51,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:53,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:29:53,852 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:29:56,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:29:56,669 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:29:56,670 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:29:56,672 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:29:56,762 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:29:57,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:29:58,777 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:29:59,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:02,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:05,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:05,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:05,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:08,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:08,842 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:30:10,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:13,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:15,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:15,894 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:30:18,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:20,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:23,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:23,853 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:30:25,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:25,960 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:30:28,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:30,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:33,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:34,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:30:35,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:38,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:42,105 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:44,075 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:30:44,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:47,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:49,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:52,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:52,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:53,852 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:30:54,154 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:30:54,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:30:57,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:30:59,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:03,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:04,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:31:05,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:08,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:10,177 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:14,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:31:14,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:15,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:20,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:20,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:20,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:20,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:23,853 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:31:24,389 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:31:25,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:25,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:30,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:31,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:34,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:31:36,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:36,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:41,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:41,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:44,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:31:46,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:47,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:51,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:52,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:31:53,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:31:54,612 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:31:57,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:31:58,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:02,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:32:03,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:04,676 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:08,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:09,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:12,823 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:32:14,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:15,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:19,418 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:20,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:22,892 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:32:23,854 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:32:24,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:25,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:29,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:31,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:32,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:32:34,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:36,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:40,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:40,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:40,191 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:32:41,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:45,102 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:46,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:50,260 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:32:50,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:51,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:32:53,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:32:55,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:32:57,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:00,339 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:01,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:03,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:07,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:07,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:08,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:08,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:33:08,578 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:33:08,578 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:33:08,580 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:33:09,398 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:33:09,400 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:10,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:12,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:13,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:17,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:18,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:19,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:22,873 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:23,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:33:24,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:27,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:30,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:30,528 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:33,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:35,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:38,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:40,577 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:40,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:43,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:45,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:49,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:50,643 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:51,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:53,855 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:33:54,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:33:56,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:33:58,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:33:59,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:02,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:05,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:07,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:08,766 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:34:10,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:12,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:15,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:16,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:34:17,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:20,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:22,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:23,858 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:34:26,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:26,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:34:28,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:31,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:33,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:36,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:36,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:34:39,559 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:42,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:44,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:47,034 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:34:48,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:50,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:53,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:53,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:34:53,858 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:34:55,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:34:57,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:34:58,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:00,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:04,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:05,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:35:06,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:09,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:11,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:14,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:15,238 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:35:16,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:19,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:21,834 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:23,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:35:25,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:35:25,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:25,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:26,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:31,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:31,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:34,355 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:35:36,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:36,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:41,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:42,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:42,415 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:35:47,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:47,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:52,479 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:35:52,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:52,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:35:53,859 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:35:57,608 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:35:57,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:02,544 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:02,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:02,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:07,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:07,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:12,611 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:13,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:13,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:18,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:19,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:20,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:36:20,403 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:36:20,404 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:36:20,407 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:36:20,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:36:22,678 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:23,683 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:23,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:36:23,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:24,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:28,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:29,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:30,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:34,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:34,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:39,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:39,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:39,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:40,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:44,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:44,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:48,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:36:50,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:50,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:53,860 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:36:55,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:36:56,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:36:58,919 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:01,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:01,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:06,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:07,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:07,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:08,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:11,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:12,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:16,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:17,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:19,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:22,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:22,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:23,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:37:27,685 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:27,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:29,107 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:32,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:32,781 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:38,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:38,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:39,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:43,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:44,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:47,227 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:48,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:49,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:53,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:53,864 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:37:54,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:54,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:37:57,290 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:37:59,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:37:59,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:04,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:04,860 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:07,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:38:09,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:09,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:15,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:38:15,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:15,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:20,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:21,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:21,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:21,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:23,862 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:38:25,476 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:38:25,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:26,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:30,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:32,500 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:35,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:38:36,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:37,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:41,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:42,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:38:42,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:46,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:47,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:52,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:52,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:38:52,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:38:53,863 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:38:57,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:38:58,128 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:02,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:03,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:04,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:08,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:09,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:09,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:12,769 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:13,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:14,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:18,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:20,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:22,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:23,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:23,863 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:39:25,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:28,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:31,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:32,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:39:32,703 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:39:32,704 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:39:32,707 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:39:32,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:39:34,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:34,938 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:35,946 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:36,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:39,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:41,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:41,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:44,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:45,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:46,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:49,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:51,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:51,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:53,864 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:39:54,865 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:39:55,068 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:39:57,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:39:59,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:03,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:05,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:40:05,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:08,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:10,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:13,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:15,189 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:40:16,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:18,362 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:21,435 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:23,864 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:40:24,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:24,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:25,244 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:40:26,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:29,150 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:31,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:34,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:35,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:40:37,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:39,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:42,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:45,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:40:45,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:47,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:52,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:52,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:52,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:52,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:40:53,865 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:40:55,427 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:40:57,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:40:58,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:02,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:03,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:05,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:41:07,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:08,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:13,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:14,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:15,550 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:41:18,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:19,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:23,865 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:41:24,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:24,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:25,625 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:41:29,388 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:29,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:35,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:35,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:35,689 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:41:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:41,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:41,722 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:41:45,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:46,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:51,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:51,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:41:51,774 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:41:53,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:41:56,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:41:56,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:00,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:01,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:02,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:06,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:07,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:10,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:11,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:13,041 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:16,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:18,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:20,979 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:21,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:23,866 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:42:23,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:28,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:28,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:31,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:33,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:34,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:38,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:38,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:40,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:41,110 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:44,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:45,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:49,153 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:42:49,154 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:42:49,155 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:42:49,157 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:42:49,173 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:42:50,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:51,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:42:51,185 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:52,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:42:53,867 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:42:55,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:42:56,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:00,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:01,249 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:43:02,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:05,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:07,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:11,317 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:43:11,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:12,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:16,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:17,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:21,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:22,578 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:23,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:43:25,408 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:43:26,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:27,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:32,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:33,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:33,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:43:37,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:38,457 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:42,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:42,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:43,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:45,553 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:43:47,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:48,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:53,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:53,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:43:53,868 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:43:53,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:43:58,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:43:58,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:03,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:03,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:44:04,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:09,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:09,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:09,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:09,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:13,734 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:44:14,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:14,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:19,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:20,183 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:21,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:44:23,869 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:44:24,869 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:25,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:30,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:30,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:31,853 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:44:35,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:35,780 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:40,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:41,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:41,912 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:44:46,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:47,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:49,961 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:44:51,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:52,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:53,869 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:44:56,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:44:57,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:57,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:44:59,022 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:45:02,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:02,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:07,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:08,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:09,089 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:45:12,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:13,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:17,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:18,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:19,150 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:45:22,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:23,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:23,871 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:45:27,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:28,677 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:29,208 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:45:32,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:35,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:37,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:45:38,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:40,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:43,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:45,920 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:47,319 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:45:48,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:50,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:53,872 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:45:54,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:45:56,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:45:57,380 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:00,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:01,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:05,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:46:05,265 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:46:05,266 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:46:05,268 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:46:05,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:46:06,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:07,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:07,451 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:08,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:11,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:12,073 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:17,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:17,517 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:17,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:22,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:23,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:23,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:46:25,571 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:27,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:28,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:32,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:33,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:35,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:38,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:38,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:43,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:44,581 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:45,697 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:48,717 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:49,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:53,873 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:46:53,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:54,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:46:55,800 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:46:58,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:46:59,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:04,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:04,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:05,861 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:47:09,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:11,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:11,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:13,911 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:47:14,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:16,167 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:19,320 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:21,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:23,874 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:47:23,976 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:47:24,874 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:26,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:29,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:31,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:34,052 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:47:35,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:36,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:40,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:42,108 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:47:42,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:45,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:47,580 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:51,272 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:47:51,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:53,503 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:47:53,874 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:47:56,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:47:58,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:01,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:48:02,810 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:03,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:07,811 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:09,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:11,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:48:13,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:14,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:19,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:19,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:19,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:48:23,875 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:48:24,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:24,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:29,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:29,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:29,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:48:30,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:34,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:35,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:39,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:48:40,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:41,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:45,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:45,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:46,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:47,642 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:48:50,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:51,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:53,877 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:48:56,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:48:57,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:48:57,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:01,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:02,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:06,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:07,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:07,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:11,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:12,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:15,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:49:15,277 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:49:15,277 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:49:15,279 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:49:15,830 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:49:15,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:17,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:17,839 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:18,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:23,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:23,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:23,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:49:25,891 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:28,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:28,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:33,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:34,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:35,943 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:39,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:39,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:44,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:44,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:46,004 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:49,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:49,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:53,878 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:49:54,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:54,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:49:56,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:49:59,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:49:59,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:04,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:50:05,769 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:05,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:10,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:11,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:14,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:50:16,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:16,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:21,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:21,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:23,878 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:50:24,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:50:26,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:26,879 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:32,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:32,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:32,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:50:37,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:37,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:42,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:50:42,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:42,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:47,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:48,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:51,437 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:50:53,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:53,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:50:53,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:50:58,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:50:58,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:03,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:03,881 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:08,575 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:08,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:09,546 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:51:13,576 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:13,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:19,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:19,608 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:51:19,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:23,879 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:51:24,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:24,880 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:29,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:29,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:51:30,430 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:35,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:36,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:39,746 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:51:40,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:41,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:45,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:46,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:46,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:49,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:51:51,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:51,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:53,880 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:51:56,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:51:57,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:57,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:51:59,878 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:02,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:02,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:07,922 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:07,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:08,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:12,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:13,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:17,985 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:18,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:18,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:23,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:23,882 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:52:25,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:28,047 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:29,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:30,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:34,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:35,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:35,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:52:35,605 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:52:35,606 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:52:35,608 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:52:36,093 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:52:36,094 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:38,104 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:39,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:40,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:44,612 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:46,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:48,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:50,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:51,383 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:52:53,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:52:56,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:52:56,275 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:52:57,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:01,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:03,643 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:06,338 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:53:06,773 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:08,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:11,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:13,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:13,927 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:16,403 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:53:17,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:18,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:22,117 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:23,883 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:53:24,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:26,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:53:27,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:29,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:32,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:34,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:53:35,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:38,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:41,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:43,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:44,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:53:46,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:48,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:51,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:53,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:53:53,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:53:53,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:53:56,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:53:58,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:02,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:03,751 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:54:04,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:07,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:09,670 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:11,809 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:54:12,774 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:14,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:19,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:19,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:21,871 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:54:23,895 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:54:24,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:25,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:29,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:29,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:31,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:31,944 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:54:34,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:36,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:40,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:42,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:54:42,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:45,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:47,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:50,072 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:54:51,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:52,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:54:53,884 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:54:57,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:54:57,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:00,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:02,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:03,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:07,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:09,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:10,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:12,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:14,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:17,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:17,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:20,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:20,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:22,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:23,885 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:55:25,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:28,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:30,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:31,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:33,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:36,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:38,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:40,410 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:41,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:43,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:46,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:46,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:55:46,983 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:55:46,983 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:55:46,986 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:55:47,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:55:48,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:48,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:51,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:53,887 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:55:54,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:55:57,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:55:58,519 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:55:59,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:02,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:05,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:06,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:56:07,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:10,123 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:12,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:15,546 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:16,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:56:17,577 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:20,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:22,579 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:23,888 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:56:26,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:26,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:56:28,402 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:32,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:33,403 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:36,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:56:37,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:38,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:43,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:44,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:44,822 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:56:48,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:49,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:53,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:53,889 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:56:54,884 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:56:54,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:56:58,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:56:59,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:03,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:03,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:03,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:57:05,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:08,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:10,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:13,847 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:14,008 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:57:16,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:20,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:21,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:23,889 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:57:24,066 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:57:25,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:26,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:30,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:30,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:32,882 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:34,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:57:35,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:37,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:40,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:42,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:46,199 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:57:46,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:46,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:48,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:51,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:53,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:57:53,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:57:56,268 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:57:56,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:57:58,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:02,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:02,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:03,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:06,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:58:07,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:08,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:12,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:14,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:14,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:58:17,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:19,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:23,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:23,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:58:24,478 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:58:24,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:28,142 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:29,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:33,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:34,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:58:35,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:38,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:40,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:42,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:58:44,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:45,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:49,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:50,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:52,666 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:58:53,891 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:58:54,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:55,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:58:59,650 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:58:59,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 02:58:59,687 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 02:58:59,688 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 02:58:59,690 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 02:58:59,719 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 02:59:00,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:01,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:04,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:06,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:09,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:10,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:11,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:14,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:16,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:20,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:20,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:20,840 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:22,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:23,892 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:59:26,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:27,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:28,899 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:31,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:32,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:37,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:37,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:38,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:38,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:42,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:43,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:47,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:48,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:49,026 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:52,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:53,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 02:59:53,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 02:59:57,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 02:59:57,648 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 02:59:58,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:03,697 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:03,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:03,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:06,148 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:00:08,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:09,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:14,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:14,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:16,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:00:19,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:19,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:23,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:00:24,265 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:00:24,793 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:24,894 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:29,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:29,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:34,328 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:00:34,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:35,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:39,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:40,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:44,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:00:45,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:45,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:50,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:00:51,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:51,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:53,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:00:54,464 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:00:56,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:00:56,895 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:01,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:02,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:01:02,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:06,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:07,543 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:11,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:12,572 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:01:12,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:16,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:17,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:22,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:22,636 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:01:23,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:23,894 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:01:28,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:28,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:30,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:01:33,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:34,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:39,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:40,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:40,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:01:44,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:45,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:50,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:50,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:01:51,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:53,895 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:01:56,419 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:01:56,896 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:01:58,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:01,420 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:02,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:06,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:02:06,424 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:02:06,425 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:02:06,428 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:02:06,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:02:07,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:08,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:08,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:12,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:13,437 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:16,998 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:17,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:18,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:23,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:23,751 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:23,896 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:02:27,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:28,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:29,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:33,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:33,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:35,127 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:35,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:38,705 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:40,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:44,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:45,205 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:46,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:49,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:51,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:02:53,897 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:02:54,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:02:55,264 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:02:56,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:00,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:02,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:03,323 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:03:05,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:07,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:10,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:12,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:03:12,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:15,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:17,772 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:20,426 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:03:20,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:20,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:22,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:23,897 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:03:25,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:27,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:30,480 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:03:31,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:33,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:37,155 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:37,156 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:38,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:40,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:03:42,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:43,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:47,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:48,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:48,604 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:03:52,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:53,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:03:53,898 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:03:57,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:03:58,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:02,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:03,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:07,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:08,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:12,269 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:13,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:17,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:18,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:22,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:23,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:04:23,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:27,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:28,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:33,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:34,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:34,858 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:04:38,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:39,846 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:43,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:44,917 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:04:45,393 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:48,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:50,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:53,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:53,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:04:53,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:04:54,997 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:04:55,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:04:58,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:00,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:03,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:05,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:05:06,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:09,055 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:11,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:13,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:05:14,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:17,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:19,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:19,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:19,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:23,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:23,172 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:05:23,899 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:05:24,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:28,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:30,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:31,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:05:34,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:35,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:39,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:40,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:41,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:05:44,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:45,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:49,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:05:49,163 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:05:49,163 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:05:49,166 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:05:49,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:05:50,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:50,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:05:51,456 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:05:53,900 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:05:55,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:05:57,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:01,522 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:01,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:02,600 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:06,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:07,808 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:07,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:08,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:11,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:12,809 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:16,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:18,031 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:18,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:22,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:23,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:23,901 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:06:27,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:28,690 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:28,718 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:32,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:33,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:38,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:38,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:39,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:39,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:43,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:44,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:46,837 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:48,776 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:49,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:53,778 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:53,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:06:54,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:54,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:06:56,901 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:06:58,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:06:59,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:03,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:04,954 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:07:05,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:09,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:10,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:14,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:15,019 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:07:16,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:19,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:21,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:21,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:23,902 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:07:24,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:25,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:07:26,886 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:29,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:32,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:33,142 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:07:35,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:37,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:40,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:42,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:43,201 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:07:45,381 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:47,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:50,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:52,438 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:07:53,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:07:53,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:07:55,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:07:58,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:01,311 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:08:01,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:03,693 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:06,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:08,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:08,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:11,399 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:08:12,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:13,912 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:17,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:19,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:19,460 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:08:22,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:23,903 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:08:24,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:27,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:29,532 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:08:29,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:32,712 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:36,050 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:37,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:39,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:08:41,051 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:42,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:46,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:47,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:51,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:52,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:08:53,905 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:08:56,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:56,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:08:57,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:08:58,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:01,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:03,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:05,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:09:05,245 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:09:05,245 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:09:05,248 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:09:05,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:09:06,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:07,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:07,780 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:08,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:13,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:13,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:16,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:18,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:18,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:23,822 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:23,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:09:24,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:24,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:26,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:29,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:29,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:34,747 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:34,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:34,984 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:39,748 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:39,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:44,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:45,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:45,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:49,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:50,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:09:53,115 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:09:53,905 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:09:55,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:09:55,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:00,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:01,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:03,193 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:10:06,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:06,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:11,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:11,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:12,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:13,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:10:16,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:17,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:21,707 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:22,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:23,318 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:10:23,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:10:26,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:27,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:31,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:10:32,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:33,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:38,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:38,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:41,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:10:43,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:43,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:48,550 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:48,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:51,508 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:10:53,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:53,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:53,906 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:10:59,085 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:10:59,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:10:59,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:04,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:04,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:09,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:09,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:09,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:14,332 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:15,434 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:17,695 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:19,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:20,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:23,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:11:24,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:25,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:25,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:27,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:30,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:30,628 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:35,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:35,824 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:36,536 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:40,607 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:41,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:45,902 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:46,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:47,063 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:52,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:11:52,126 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:53,908 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:11:55,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:11:57,127 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:11:57,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:02,322 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:03,013 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:03,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:07,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:08,374 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:11,779 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:12:11,781 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:12:11,782 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:12:11,784 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:12:12,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:12:12,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:12,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:13,076 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:13,786 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:14,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:17,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:18,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:22,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:23,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:23,909 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:12:27,164 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:27,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:28,829 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:33,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:34,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:37,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:39,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:39,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:44,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:44,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:44,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:45,298 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:49,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:49,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:53,909 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:12:54,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:54,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:12:55,354 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:12:59,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:12:59,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:03,406 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:13:04,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:05,864 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:10,866 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:13,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:13:16,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:16,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:21,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:21,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:23,603 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:13:23,918 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:13:26,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:26,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:31,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:32,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:33,663 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:13:36,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:37,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:41,720 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:13:42,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:42,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:47,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:48,615 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:51,804 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:13:52,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:53,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:53,934 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:13:57,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:13:58,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:13:58,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:01,873 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:03,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:03,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:09,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:09,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:09,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:14,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:14,132 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:19,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:19,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:19,999 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:23,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:14:24,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:25,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:28,055 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:30,450 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:30,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:35,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:36,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:38,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:40,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:41,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:46,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:46,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:46,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:51,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:52,823 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:14:53,920 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:14:55,233 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:14:57,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:14:57,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:02,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:03,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:05,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:15:07,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:08,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:12,093 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:13,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:13,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:15,374 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:15:17,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:18,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:22,464 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:22,843 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:15:22,844 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:15:22,845 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:15:22,847 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:15:23,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:15:23,455 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:15:23,784 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:23,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:15:27,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:28,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:33,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:33,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:15:33,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:38,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:40,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:43,159 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:43,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:15:45,104 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:48,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:50,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:51,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:15:53,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:15:53,921 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:15:55,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:15:58,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:00,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:01,702 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:03,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:05,851 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:08,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:09,756 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:11,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:14,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:17,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:19,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:19,829 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:22,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:23,922 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:16:24,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:27,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:29,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:30,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:32,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:35,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:38,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:39,952 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:41,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:43,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:46,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:48,000 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:48,468 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:51,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:53,923 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:16:54,661 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:16:56,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:16:58,070 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:16:59,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:01,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:05,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:07,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:08,126 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:17:10,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:12,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:15,544 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:16,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:17:17,818 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:20,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:22,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:23,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:17:26,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:26,252 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:17:28,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:32,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:33,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:36,324 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:17:37,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:38,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:43,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:44,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:44,384 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:17:48,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:49,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:53,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:53,449 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:17:53,934 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:17:54,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:17:58,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:17:59,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:03,515 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:18:03,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:05,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:09,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:10,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:11,568 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:18:14,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:16,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:20,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:21,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:21,635 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:18:23,924 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:18:25,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:26,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:29,692 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:18:29,693 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:18:29,695 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:18:29,698 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:18:29,703 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:18:30,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:31,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:18:32,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:35,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:37,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:39,770 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:18:41,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:43,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:47,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:47,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:48,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:49,831 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:18:52,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:53,508 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:53,925 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:18:57,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:18:58,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:18:59,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:02,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:03,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:07,948 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:07,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:09,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:12,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:14,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:18,007 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:18,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:18,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:19,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:23,926 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:19:24,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:24,928 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:26,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:29,331 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:30,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:34,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:35,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:36,124 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:39,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:40,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:44,830 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:45,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:46,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:49,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:50,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:19:53,927 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:19:54,250 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:19:55,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:19:56,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:01,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:01,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:04,315 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:20:06,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:07,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:12,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:13,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:14,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:20:17,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:18,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:22,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:23,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:23,927 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:20:24,448 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:20:27,484 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:28,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:32,501 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:20:32,992 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:34,046 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:38,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:39,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:42,559 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:20:43,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:44,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:49,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:49,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:50,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:50,619 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:20:53,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:20:54,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:55,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:20:59,482 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:20:59,685 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:01,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:04,483 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:06,594 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:09,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:10,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:12,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:16,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:16,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:17,456 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:19,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:21,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:22,884 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:23,928 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:21:27,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:27,903 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:28,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:32,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:33,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:36,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:21:36,350 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:21:36,351 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:21:36,353 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:21:36,959 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:21:37,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:37,965 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:38,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:42,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:43,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:47,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:48,036 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:48,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:53,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:53,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:21:53,929 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:21:56,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:21:58,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:21:58,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:03,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:03,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:03,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:06,165 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:22:08,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:09,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:14,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:14,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:14,222 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:22:19,076 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:19,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:23,930 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:22:24,095 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:24,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:22:24,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:29,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:30,338 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:32,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:22:35,353 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:35,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:40,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:40,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:42,495 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:22:45,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:45,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:50,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:22:50,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:52,560 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:22:53,930 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:22:55,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:22:55,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:00,814 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:00,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:02,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:05,815 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:05,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:10,682 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:11,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:11,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:16,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:17,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:17,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:20,753 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:21,708 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:22,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:23,931 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:23:26,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:27,763 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:30,810 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:32,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:32,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:37,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:38,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:38,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:43,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:43,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:47,957 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:48,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:48,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:53,358 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:53,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:23:53,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:23:58,018 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:23:59,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:23:59,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:04,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:04,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:04,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:08,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:24:09,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:10,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:15,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:15,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:16,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:24:20,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:20,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:23,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:24:25,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:25,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:26,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:24:30,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:30,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:34,285 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:24:35,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:35,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:40,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:40,360 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:43,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:24:43,366 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:24:43,367 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:24:43,369 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:24:44,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:24:44,352 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:24:45,365 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:45,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:45,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:50,373 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:51,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:24:53,933 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:24:54,414 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:24:55,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:24:56,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:00,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:02,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:02,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:04,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:25:06,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:07,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:12,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:12,385 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:12,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:25:17,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:17,386 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:22,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:22,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:25:23,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:23,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:25:27,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:28,802 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:30,672 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:25:33,804 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:33,840 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:38,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:38,915 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:40,731 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:25:43,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:44,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:49,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:49,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:50,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:25:53,934 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:25:54,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:54,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:25:59,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:25:59,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:04,436 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:04,896 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:26:05,510 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:10,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:10,555 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:12,953 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:26:15,549 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:15,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:20,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:21,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:21,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:22,134 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:26:23,935 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:26:25,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:26,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:31,411 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:31,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:32,196 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:26:36,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:36,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:41,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:42,024 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:42,269 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:26:47,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:48,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:48,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:48,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:50,326 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:26:52,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:53,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:26:53,935 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:26:57,937 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:26:58,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:00,396 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:02,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:03,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:07,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:08,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:10,462 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:13,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:13,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:18,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:18,525 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:19,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:23,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:23,937 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:27:25,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:28,592 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:29,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:30,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:34,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:36,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:36,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:40,199 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:41,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:45,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:46,364 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:46,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:50,463 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:52,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:52,451 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:27:53,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:27:53,496 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:27:53,496 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:27:53,499 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:27:53,767 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:27:53,937 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:27:54,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:27:55,938 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:27:57,452 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:00,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:02,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:04,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:28:06,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:07,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:11,743 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:13,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:14,930 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:28:17,573 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:18,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:22,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:28:23,548 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:23,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:28:24,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:28,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:29,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:33,061 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:28:34,767 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:34,931 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:39,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:40,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:40,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:43,131 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:28:45,065 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:45,200 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:50,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:50,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:28:51,180 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:28:53,938 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:28:55,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:28:55,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:01,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:01,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:01,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:06,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:07,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:07,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:10,314 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:11,469 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:12,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:16,471 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:18,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:18,377 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:22,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:23,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:23,939 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:29:27,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:28,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:28,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:32,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:33,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:37,350 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:38,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:38,520 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:42,351 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:44,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:46,578 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:47,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:49,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:52,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:29:53,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:29:55,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:55,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:29:56,651 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:29:58,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:00,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:03,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:05,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:06,721 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:30:08,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:10,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:13,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:14,785 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:30:15,563 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:18,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:21,701 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:21,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:21,704 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:23,850 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:23,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:30:24,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:30:26,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:28,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:32,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:34,317 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:34,918 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:30:37,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:39,319 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:42,970 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:30:43,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:45,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:48,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:50,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:53,030 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:30:53,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:30:53,941 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:30:55,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:30:59,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:00,657 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:31:00,659 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:31:00,659 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:31:00,662 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:31:01,083 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:31:01,663 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:03,091 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:31:04,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:06,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:09,753 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:11,151 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:31:11,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:14,754 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:16,855 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:20,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:21,216 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:31:22,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:23,942 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:31:25,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:27,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:30,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:31,276 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:31:33,377 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:35,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:39,099 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:39,333 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:31:40,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:40,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:44,100 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:46,091 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:49,118 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:49,402 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:31:51,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:53,943 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:31:54,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:31:56,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:56,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:31:57,463 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:00,340 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:01,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:05,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:06,506 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:06,526 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:10,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:11,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:15,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:16,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:16,733 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:20,759 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:23,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:23,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:23,943 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:32:24,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:26,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:28,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:31,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:33,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:34,704 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:36,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:38,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:38,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:41,785 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:44,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:46,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:47,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:49,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:52,642 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:53,944 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:32:54,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:32:54,834 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:32:57,945 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:32:59,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:02,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:04,897 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:33:05,064 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:08,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:10,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:13,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:14,962 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:33:15,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:18,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:21,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:23,011 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:33:23,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:23,944 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:33:26,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:28,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:31,658 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:33,073 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:33:34,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:36,659 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:39,680 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:41,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:41,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:43,133 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:33:44,768 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:46,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:49,770 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:51,194 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:33:51,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:53,945 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:33:54,946 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:33:58,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:33:58,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:00,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:01,308 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:03,406 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:05,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:08,824 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:08,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:34:08,872 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:34:08,873 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:34:08,875 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:34:09,363 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:34:10,877 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:11,376 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:13,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:15,878 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:19,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:19,441 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:21,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:23,946 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:34:24,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:26,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:29,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:29,510 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:32,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:35,442 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:37,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:39,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:40,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:42,606 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:45,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:47,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:47,921 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:50,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:52,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:34:53,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:34:56,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:34:56,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:34:58,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:01,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:03,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:04,754 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:35:07,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:08,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:13,160 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:13,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:14,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:14,817 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:35:18,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:19,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:22,866 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:35:23,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:23,946 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:35:24,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:28,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:29,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:32,927 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:35:33,369 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:35,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:38,372 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:40,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:42,996 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:35:44,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:44,116 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:46,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:49,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:51,057 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:35:51,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:53,947 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:35:54,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:56,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:35:59,755 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:35:59,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:01,121 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:36:02,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:04,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:07,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:09,765 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:11,184 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:36:12,831 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:14,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:17,832 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:19,242 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:36:19,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:22,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:23,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:36:26,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:26,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:26,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:28,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:29,309 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:36:31,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:33,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:37,047 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:39,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:39,390 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:36:42,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:44,198 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:47,584 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:49,489 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:36:49,876 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:52,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:53,948 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:36:54,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:36:58,069 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:36:59,548 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:00,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:03,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:05,341 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:07,615 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:08,517 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:10,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:13,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:15,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:16,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:37:16,210 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:37:16,210 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:37:16,212 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:37:16,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:37:17,679 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:19,180 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:21,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:23,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:37:24,720 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:26,844 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:27,743 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:29,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:31,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:34,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:36,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:37,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:39,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:41,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:45,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:45,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:45,876 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:47,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:50,078 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:52,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:37:53,950 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:37:54,932 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:37:55,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:37:57,951 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:01,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:02,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:03,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:06,629 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:08,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:12,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:12,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:13,060 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:14,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:17,329 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:19,428 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:22,636 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:23,122 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:23,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:38:24,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:27,637 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:29,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:32,916 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:33,190 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:35,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:39,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:40,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:41,254 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:44,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:45,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:49,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:50,309 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:51,321 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:53,951 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:38:54,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:38:55,952 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:38:59,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:38:59,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:01,519 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:04,416 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:06,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:09,445 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:39:10,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:12,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:16,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:16,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:17,185 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:19,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:39:21,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:22,352 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:23,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:39:26,647 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:27,561 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:39:28,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:31,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:33,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:37,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:37,667 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:39:38,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:42,325 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:43,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:47,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:48,523 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:51,760 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:39:52,532 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:53,952 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:39:53,953 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:39:57,533 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:39:58,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:02,534 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:03,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:07,535 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:08,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:13,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:13,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:18,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:18,957 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:23,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:23,953 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:40:24,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:25,971 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:40:28,631 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:29,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:33,644 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:34,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:36,042 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:40:38,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:39,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:43,841 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:44,906 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:46,114 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:40:48,842 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:49,907 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:51,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:40:51,977 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:40:51,977 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:40:51,979 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:40:52,157 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:40:53,163 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:40:53,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:40:53,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:40:54,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:40:59,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:00,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:00,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:03,223 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:41:05,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:05,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:10,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:10,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:11,277 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:41:15,439 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:15,587 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:20,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:21,335 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:41:21,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:23,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:41:25,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:26,956 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:31,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:31,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:31,392 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:41:32,495 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:37,496 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:37,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:41,453 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:41:42,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:42,848 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:47,849 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:47,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:47,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:49,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:41:52,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:53,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:53,955 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:41:58,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:58,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:41:58,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:41:59,585 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:42:03,321 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:03,409 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:07,641 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:42:08,465 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:08,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:13,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:13,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:17,713 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:42:18,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:18,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:23,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:23,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:42:24,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:27,776 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:42:29,157 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:29,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:34,158 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:35,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:37,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:42:39,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:40,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:45,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:45,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:45,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:42:50,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:50,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:53,957 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:42:55,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:42:55,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:42:55,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:00,397 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:00,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:06,035 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:06,154 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:06,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:11,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:12,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:16,098 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:16,649 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:17,571 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:21,651 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:22,867 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:23,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:43:26,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:26,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:27,868 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:31,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:32,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:32,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:34,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:36,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:37,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:42,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:43,184 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:44,294 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:47,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:49,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:52,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:52,347 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:43:53,958 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:43:54,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:57,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:43:59,394 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:59,395 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:43:59,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:43:59,451 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:43:59,451 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:43:59,457 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:44:00,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:44:01,397 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:44:03,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:04,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:08,460 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:09,444 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:44:09,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:13,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:14,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:19,023 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:19,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:44:19,883 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:23,959 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:44:24,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:26,129 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:29,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:44:30,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:31,130 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:35,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:36,488 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:37,626 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:44:40,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:41,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:45,700 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:47,139 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:47,692 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:44:51,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:52,140 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:53,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:44:56,346 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:44:57,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:44:57,763 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:01,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:03,479 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:05,820 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:06,616 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:08,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:11,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:13,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:15,890 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:16,726 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:18,673 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:21,727 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:23,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:23,940 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:23,960 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:45:26,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:28,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:31,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:34,009 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:34,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:37,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:40,201 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:43,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:44,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:45,202 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:48,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:50,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:52,129 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:45:53,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:45:53,961 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:45:55,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:45:58,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:00,421 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:02,209 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:04,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:05,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:09,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:10,261 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:10,731 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:14,911 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:16,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:16,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:20,325 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:20,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:21,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:23,962 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:46:25,964 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:27,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:28,378 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:31,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:32,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:37,601 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:38,446 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:38,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:42,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:45,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:45,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:48,101 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:48,516 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:50,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:53,103 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:46:53,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:46:55,119 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:46:56,574 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:46:58,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:00,120 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:03,166 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:04,361 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:47:04,362 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:47:04,362 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:47:04,364 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:47:04,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:47:05,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:05,645 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:47:08,366 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:10,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:13,367 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:15,422 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:15,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:47:18,827 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:21,709 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:23,773 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:47:23,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:47:23,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:26,710 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:28,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:32,595 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:32,596 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:33,845 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:47:34,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:37,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:39,641 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:41,910 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:47:42,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:45,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:49,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:50,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:51,973 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:47:53,965 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:47:54,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:55,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:47:59,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:47:59,795 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:00,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:02,048 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:48:04,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:05,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:10,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:10,101 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:48:11,090 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:15,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:16,092 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:20,166 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:48:20,429 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:21,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:23,966 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:48:26,540 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:26,967 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:30,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:48:31,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:32,613 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:36,585 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:37,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:38,281 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:48:41,586 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:42,645 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:46,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:48,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:48,346 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:48:51,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:53,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:53,967 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:48:57,816 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:48:58,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:48:58,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:03,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:04,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:04,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:06,477 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:08,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:09,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:13,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:14,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:16,545 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:18,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:19,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:23,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:23,967 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:49:24,719 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:26,607 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:28,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:29,721 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:34,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:34,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:35,602 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:39,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:40,603 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:44,682 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:44,729 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:45,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:49,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:51,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:53,967 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:49:54,796 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:49:54,968 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:49:56,415 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:49:59,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:01,541 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:01,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:05,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:06,542 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:09,756 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:50:09,757 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:50:09,758 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:50:09,760 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:50:09,909 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:50:10,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:11,683 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:11,924 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:15,762 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:17,899 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:17,900 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:19,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:20,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:22,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:23,968 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:50:25,969 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:27,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:30,058 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:31,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:32,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:36,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:38,112 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:38,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:41,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:43,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:46,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:48,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:48,179 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:52,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:53,145 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:50:53,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:50:57,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:50:58,248 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:50:58,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:02,481 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:04,354 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:06,301 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:51:08,368 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:09,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:13,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:14,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:16,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:51:18,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:19,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:23,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:23,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:51:25,066 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:26,438 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:51:29,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:30,067 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:34,356 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:34,499 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:51:35,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:39,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:41,412 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:44,507 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:44,569 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:51:46,413 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:49,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:52,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:52,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:53,970 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:51:54,639 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:51:54,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:51:57,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:51:59,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:02,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:02,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:52:05,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:07,398 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:10,531 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:12,794 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:52:13,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:16,165 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:19,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:19,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:21,401 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:22,854 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:52:23,971 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:52:24,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:26,972 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:30,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:30,907 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:52:32,230 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:35,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:37,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:40,968 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:52:41,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:42,334 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:46,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:47,335 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:49,024 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:52:51,493 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:52,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:53,973 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:52:56,689 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:52:57,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:52:58,096 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:01,690 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:02,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:06,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:08,161 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:08,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:11,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:13,162 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:15,678 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:53:15,680 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:53:15,680 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:53:15,682 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:53:16,245 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:53:16,246 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:17,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:18,684 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:23,630 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:23,686 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:23,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:53:26,300 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:28,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:29,634 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:33,861 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:33,862 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:35,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:36,367 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:38,863 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:40,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:44,323 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:44,417 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:45,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:49,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:50,355 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:53:53,974 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:53:54,506 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:53:55,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:53:55,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:00,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:00,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:05,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:05,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:05,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:08,598 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:54:10,562 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:11,633 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:15,653 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:16,665 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:54:16,737 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:20,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:21,738 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:23,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:54:25,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:26,727 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:54:27,672 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:30,716 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:32,674 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:35,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:36,791 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:54:37,960 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:40,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:42,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:44,849 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:54:46,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:48,509 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:51,537 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:51,538 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:53,545 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:54:53,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:54:54,920 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:54:56,539 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:54:58,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:02,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:02,983 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:55:04,497 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:07,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:09,499 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:13,053 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:55:13,444 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:14,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:18,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:18,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:19,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:21,102 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:55:23,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:23,976 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:55:24,993 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:28,904 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:30,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:31,175 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:55:33,905 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:35,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:39,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:40,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:41,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:55:44,348 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:45,349 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:49,291 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:55:49,417 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:50,505 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:53,979 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:55:55,604 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:55:55,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:55:58,342 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:00,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:01,655 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:05,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:05,805 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:06,394 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:06,940 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:10,806 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:11,941 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:16,196 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:16,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:17,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:21,197 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:22,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:23,993 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:56:24,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:56:24,795 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:56:24,796 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:56:24,799 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:56:25,524 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:56:26,529 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:26,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:27,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:32,803 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:32,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:36,591 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:37,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:38,113 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:43,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:43,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:44,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:48,407 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:48,617 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:53,695 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:53,859 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:56:53,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:56:54,712 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:56:58,696 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:56:58,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:03,958 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:04,131 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:04,781 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:57:08,959 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:09,133 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:12,844 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:57:14,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:14,856 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:19,857 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:20,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:20,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:22,921 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:57:23,980 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:57:24,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:25,194 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:29,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:31,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:32,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:57:35,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:36,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:41,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:57:41,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:41,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:46,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:47,343 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:51,176 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:57:51,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:52,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:53,981 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:57:56,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:57:57,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:57:59,235 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:58:02,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:02,933 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:08,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:08,344 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:09,305 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:58:13,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:13,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:19,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:19,339 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:19,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:58:23,983 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:58:24,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:24,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:29,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:29,985 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:34,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:34,986 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:39,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:39,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:44,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:44,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:48,570 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:58:50,089 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:50,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:53,982 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:58:55,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:58:56,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:58:56,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:00,646 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:01,679 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:01,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:06,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:06,852 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:06,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:06,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:08,714 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:11,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:11,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:16,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:17,404 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:18,790 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:22,405 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:22,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:23,984 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:59:27,560 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:27,925 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:28,872 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:32,561 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:32,926 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:37,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:38,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:38,939 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:43,591 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:44,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:47,003 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:48,788 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:49,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:53,789 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:53,985 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 03:59:54,934 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:54,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 03:59:55,115 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 03:59:55,116 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 03:59:55,116 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 03:59:55,119 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 03:59:56,074 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 03:59:57,080 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 03:59:59,121 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 03:59:59,935 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:04,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:05,136 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:00:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:09,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:10,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:14,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:15,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:17,217 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:00:19,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:20,565 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:23,986 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:00:24,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:25,566 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:29,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:30,567 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:34,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:35,568 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:39,990 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:40,570 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:45,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:45,386 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:00:46,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:50,029 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:52,501 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:52,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:00:53,986 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:00:55,450 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:00:55,987 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:00:57,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:00,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:02,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:05,518 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:06,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:07,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:11,195 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:13,203 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:13,583 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:16,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:18,204 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:22,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:23,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:23,655 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:23,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:01:27,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:29,687 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:33,664 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:33,717 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:34,694 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:38,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:40,639 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:41,768 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:43,711 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:45,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:48,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:50,827 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:50,845 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:53,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:01:53,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:01:56,887 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:56,888 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:01:58,880 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:01:58,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:01,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:03,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:06,889 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:08,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:11,898 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:13,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:17,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:18,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:21,023 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:02:22,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:23,987 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:02:23,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:28,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:28,149 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:28,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:31,087 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:02:33,151 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:34,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:38,640 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:40,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:41,146 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:02:44,134 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:45,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:49,197 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:02:49,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:50,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:53,988 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:02:54,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:02:55,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:02:59,257 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:02:59,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:01,551 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:05,502 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:06,552 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:07,310 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:10,504 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:11,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:15,698 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:16,825 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:17,366 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:20,699 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:21,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:23,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:03:26,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:26,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:27,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:27,432 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:31,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:32,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:34,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:03:34,247 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:03:34,247 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:03:34,250 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:03:34,490 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:03:35,494 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:37,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:37,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:38,253 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:42,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:42,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:43,254 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:45,573 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:47,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:48,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:52,432 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:53,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:03:53,631 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:03:53,990 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:03:57,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:03:58,991 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:02,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:03,698 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:04:04,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:07,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:09,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:12,966 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:13,765 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:04:15,081 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:19,086 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:20,082 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:21,816 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:04:24,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:04:24,088 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:26,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:29,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:29,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:31,893 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:04:31,929 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:34,467 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:36,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:39,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:41,963 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:04:42,794 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:44,583 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:47,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:49,917 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:51,028 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:04:53,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:53,991 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:04:54,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:04:58,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:04:59,084 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:00,516 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:03,582 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:07,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:09,070 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:09,153 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:12,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:14,071 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:17,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:19,218 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:19,382 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:22,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:23,992 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:05:24,994 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:27,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:29,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:30,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:32,838 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:35,984 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:37,343 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:38,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:41,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:44,441 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:46,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:47,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:49,443 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:51,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:53,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:05:54,749 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:05:56,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:05:57,471 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:05:59,750 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:01,875 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:04,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:07,535 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:07,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:09,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:12,872 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:14,961 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:14,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:15,587 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:18,125 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:19,962 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:23,141 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:23,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:06:25,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:25,656 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:28,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:31,619 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:33,741 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:34,556 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:36,620 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:39,558 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:41,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:41,930 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:06:41,931 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:06:41,932 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:06:41,934 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:06:42,792 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:06:42,793 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:43,799 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:44,936 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:46,783 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:49,965 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:52,036 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:53,864 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:06:53,994 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:06:54,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:06:57,037 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:06:59,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:02,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:03,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:07:05,472 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:07,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:10,474 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:13,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:14,010 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:07:15,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:19,137 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:21,330 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:22,063 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:07:23,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:07:24,138 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:26,997 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:29,425 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:29,426 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:32,128 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:07:32,520 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:34,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:37,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:39,791 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:40,188 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:07:42,819 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:44,792 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:47,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:49,253 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:07:50,030 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:53,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:53,995 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:07:56,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:56,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:56,449 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:07:58,597 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:07:59,322 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:08:01,462 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:03,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:06,609 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:07,379 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:08:08,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:11,610 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:13,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:17,440 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:08:17,605 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:18,745 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:23,746 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:23,870 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:23,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:08:25,492 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:08:28,871 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:29,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:34,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:34,529 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:35,547 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:08:39,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:39,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:44,487 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:44,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:49,628 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:08:50,135 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:50,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:53,997 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:08:55,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:08:55,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:08:59,706 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:00,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:01,691 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:06,621 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:07,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:07,771 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:11,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:12,622 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:17,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:17,833 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:18,453 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:22,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:23,455 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:23,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:09:27,660 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:27,898 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:28,988 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:33,989 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:34,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:37,958 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:39,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:39,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:44,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:45,147 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:45,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:46,014 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:49,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:50,148 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:53,193 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:09:53,194 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:09:53,194 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:09:53,197 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:09:53,998 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:09:54,071 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:09:55,077 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:09:55,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:09:55,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:09:56,081 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:10:00,170 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:00,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:05,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:05,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:10,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:10,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:15,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:15,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:20,173 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:20,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:23,999 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:10:25,174 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:26,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:30,175 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:31,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:35,176 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:36,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:40,186 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:41,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:45,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:46,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:50,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:51,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:10:54,003 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:10:55,188 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:10:56,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:00,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:01,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:05,190 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:06,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:10,191 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:11,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:15,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:16,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:20,205 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:21,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:24,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:11:25,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:27,001 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:30,206 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:32,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:35,207 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:37,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:40,208 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:42,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:45,209 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:47,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:50,210 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:52,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:11:54,000 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:11:55,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:11:58,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:00,211 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:03,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:05,212 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:08,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:10,213 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:13,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:15,214 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:18,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:20,215 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:23,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:24,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:12:25,216 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:29,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:30,217 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:34,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:35,218 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:39,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:40,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:44,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:45,219 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:49,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:50,220 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:12:54,002 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:12:55,002 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:12:55,221 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:00,003 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:00,222 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:05,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:05,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:10,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:10,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:15,005 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:15,225 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:20,006 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:20,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:24,016 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:13:25,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:25,226 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:30,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:30,227 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:35,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:35,228 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:40,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:40,229 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:45,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:45,231 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:50,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:13:50,232 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:54,006 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:13:55,233 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:13:56,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:00,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:01,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:05,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:06,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:10,235 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:11,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:15,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:16,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:20,237 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:21,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:24,006 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:14:25,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:27,007 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:30,239 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:32,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:35,240 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:37,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:40,241 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:42,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:45,242 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:47,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:50,243 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:52,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:14:54,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:14:55,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:14:58,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:00,245 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:03,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:05,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:08,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:10,246 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:13,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:15,255 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:18,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:20,256 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:23,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:24,007 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:15:25,257 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:29,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:30,258 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:34,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:35,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:39,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:40,259 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:44,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:45,260 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:49,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:50,261 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:15:54,008 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:15:55,008 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:15:55,262 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:00,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:00,263 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:05,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:05,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:10,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:10,264 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:15,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:15,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:20,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:20,266 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:24,008 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:16:25,267 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:26,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:30,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:31,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:35,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:36,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:40,271 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:41,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:45,272 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:46,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:50,273 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:51,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:16:54,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:16:55,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:16:57,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:00,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:02,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:05,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:07,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:10,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:12,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:15,277 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:17,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:20,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:22,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:24,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:17:25,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:28,010 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:30,279 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:33,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:35,280 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:38,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:40,281 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:43,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:45,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:48,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:50,282 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:53,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:17:54,009 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:17:55,283 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:17:59,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:00,284 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:04,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:05,285 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:09,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:10,286 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:14,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:15,287 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:19,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:20,288 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:24,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:18:25,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:25,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:30,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:30,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:35,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:35,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:40,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:40,291 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:45,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:45,292 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:50,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:18:50,293 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:54,010 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:18:55,294 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:18:56,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:00,295 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:01,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:05,296 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:06,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:10,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:11,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:15,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:16,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:20,298 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:21,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:24,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:19:25,299 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:27,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:30,300 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:32,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:35,301 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:37,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:40,302 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:42,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:45,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:47,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:50,303 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:52,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:54,011 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:19:55,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:19:57,614 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:19:57,618 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:19:57,618 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:19:57,618 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:19:57,621 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:19:57,836 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:19:58,841 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:19:59,846 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:20:00,311 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:02,623 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:05,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:07,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:10,312 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:12,625 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:15,313 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:17,626 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:20,314 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:22,627 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:24,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:20:25,315 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:28,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:30,342 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:32,990 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:20:33,249 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:35,635 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:38,250 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:41,044 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:43,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:46,858 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:46,890 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:48,251 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:53,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:53,252 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:20:54,012 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:20:58,347 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:20:59,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:03,950 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:04,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:09,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:09,480 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:14,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:16,042 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:16,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:16,043 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:18,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:21:19,122 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:19,232 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:21:24,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:21:24,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:24,359 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:26,271 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:21:29,376 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:31,932 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:34,378 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:39,379 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:44,380 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:47,939 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:47,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:47,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:49,999 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:50,395 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:21:52,973 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:21:54,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:21:55,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:21:57,974 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:00,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:02,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:05,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:07,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:10,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:12,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:15,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:17,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:20,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:22,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:24,013 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:22:26,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:27,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:31,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:32,979 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:36,016 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:37,980 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:41,017 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:42,981 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:46,223 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:46,624 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:22:47,982 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:51,224 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:52,983 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:22:54,014 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:22:56,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:22:58,696 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:22:58,908 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:01,236 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:05,056 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:05,057 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:06,238 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:08,758 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:23:10,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:11,278 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:15,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:16,818 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:23:17,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:20,740 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:22,048 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:24,015 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:23:26,494 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:26,877 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:23:27,588 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:31,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:32,589 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:36,936 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:23:37,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:38,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:42,053 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:43,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:46,993 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:23:47,297 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:49,427 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:53,901 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:53,902 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:23:54,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:23:55,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:23:57,056 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:23:58,903 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:00,025 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:04,052 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:05,408 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:07,118 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:24:09,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:10,414 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:14,702 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:15,169 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:24:15,764 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:19,703 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:20,766 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:24,017 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:24:25,187 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:25,228 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:24:26,018 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:30,189 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:31,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:35,760 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:36,466 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:37,387 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:24:40,761 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:41,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:46,068 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:47,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:47,454 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:24:52,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:24:52,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:52,513 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:54,020 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:24:55,507 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:24:57,514 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:24:58,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:03,022 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:03,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:05,567 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:25:08,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:08,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:13,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:13,491 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:14,633 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:25:18,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:18,492 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:22,684 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:25:23,739 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:23,757 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:23,758 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:24,018 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:25:29,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:30,061 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:32,745 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:25:34,077 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:35,062 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:39,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:40,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:42,815 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:25:44,289 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:45,885 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:49,290 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:51,058 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:25:52,874 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:25:54,019 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:25:55,019 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:25:56,059 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:00,021 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:01,072 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:26:01,076 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:26:01,076 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:26:01,078 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:26:01,923 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:26:01,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:02,928 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:26:05,080 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:06,995 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:10,083 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:12,192 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:12,991 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:26:15,357 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:18,592 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:20,787 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:21,044 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:26:23,593 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:24,019 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:26:26,020 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:28,975 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:31,105 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:26:31,171 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:33,976 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:36,172 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:39,178 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:41,167 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:26:41,336 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:44,179 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:46,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:49,598 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:51,231 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:26:51,837 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:54,022 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:26:54,599 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:26:57,026 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:26:59,286 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:00,108 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:02,268 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:06,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:06,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:07,270 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:09,350 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:11,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:12,485 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:16,668 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:17,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:19,405 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:21,669 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:22,924 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:24,023 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:27:27,458 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:28,528 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:29,466 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:32,459 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:33,530 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:38,247 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:38,798 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:39,584 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:43,248 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:43,799 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:48,666 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:49,650 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:49,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:53,667 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:27:54,025 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:27:55,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:27:57,715 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:27:59,265 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:00,307 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:04,370 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:05,308 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:07,786 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:28:09,371 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:10,922 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:14,918 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:15,923 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:16,848 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:28:19,919 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:21,004 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:24,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:28:25,475 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:25,476 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:26,526 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:26,913 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:28:30,477 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:31,527 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:35,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:36,713 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:36,980 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:28:40,624 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:41,715 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:45,039 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:28:46,013 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:47,014 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:52,015 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:52,143 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:28:54,026 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:28:55,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:28:57,027 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:28:57,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:02,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:02,518 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:05,170 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:07,547 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:07,675 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:12,440 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:29:12,442 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:29:12,442 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:29:12,444 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:29:13,219 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:29:13,220 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:13,316 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:13,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:14,224 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:18,327 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:18,448 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:23,287 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:23,892 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:23,947 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:24,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:29:29,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:29,998 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:33,351 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:34,106 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:35,000 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:39,107 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:40,274 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:40,275 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:41,413 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:44,423 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:45,276 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:49,424 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:50,722 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:29:51,482 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:29:54,027 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:29:55,028 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:29:55,723 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:00,796 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:00,800 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:01,534 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:05,797 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:05,801 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:11,569 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:11,599 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:11,741 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:16,742 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:16,954 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:19,654 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:21,955 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:22,097 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:24,028 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:30:27,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:27,304 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:27,389 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:29,724 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:32,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:32,390 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:37,489 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:37,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:37,783 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:42,490 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:42,646 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:47,820 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:47,851 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:47,970 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:52,821 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:52,971 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:30:54,030 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:30:57,916 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:30:58,111 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:30:58,431 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:03,114 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:03,433 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:07,988 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:31:08,564 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:08,948 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:13,949 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:14,109 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:17,045 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:31:19,110 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:19,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:24,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:31:24,553 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:24,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:24,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:25,095 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:31:29,554 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:29,724 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:34,909 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:35,038 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:35,171 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:31:39,910 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:40,039 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:45,074 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:45,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:45,240 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:31:50,075 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:50,136 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:31:54,030 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:31:55,244 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:31:55,304 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:31:56,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:01,035 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:01,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:03,353 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:06,324 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:06,337 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:11,326 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:11,813 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:13,409 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:16,817 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:16,833 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:21,144 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: partial_history -2023-11-21 04:32:21,147 DEBUG SenderThread:607337 [sender.py:send():380] send: history -2023-11-21 04:32:21,148 DEBUG SenderThread:607337 [sender.py:send_request():407] send_request: summary_record -2023-11-21 04:32:21,150 INFO SenderThread:607337 [sender.py:_save_file():1380] saving file wandb-summary.json with policy end -2023-11-21 04:32:21,465 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json -2023-11-21 04:32:22,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:22,152 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:22,469 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:23,474 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:24,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:32:27,012 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:28,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:31,539 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:32,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:33,391 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:37,328 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:38,392 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:41,606 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:42,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:42,522 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:43,652 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:48,654 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:48,942 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:51,671 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:32:53,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:32:54,031 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:32:54,032 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:59,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:32:59,305 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:01,742 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:33:04,306 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:04,498 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:09,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:09,521 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:09,943 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:11,807 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:33:14,782 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:14,944 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:19,856 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:33:19,977 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:20,079 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:24,032 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:33:24,978 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:26,033 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:29,929 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:33:30,725 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:31,853 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:36,854 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:37,009 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:39,986 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:33:42,011 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:42,168 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:47,169 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:47,234 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:50,041 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:33:52,345 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:52,839 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:54,033 DEBUG SenderThread:607337 [sender.py:send():380] send: stats -2023-11-21 04:33:58,034 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:33:58,511 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:33:58,512 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:34:00,109 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log -2023-11-21 04:34:03,515 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:34:03,826 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:34:09,396 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: status_report -2023-11-21 04:34:09,447 DEBUG HandlerThread:607337 [handler.py:handle_request():146] handle_request: keepalive -2023-11-21 04:34:10,213 INFO Thread-12 :607337 [dir_watcher.py:_on_file_modified():288] file/dir modified: /mnt/e/wandb/run-20231119_105908-nwx0xdmm/files/output.log +version https://git-lfs.github.com/spec/v1 +oid sha256:767d6af650a36dd031a87328c1273e7d84414ff2f853dd39039645637d8f2e2a +size 14610797